diff --git a/.buildkite/test_areas/model_runner_v2.yaml b/.buildkite/test_areas/model_runner_v2.yaml index eaec866f58..0b47c8f545 100644 --- a/.buildkite/test_areas/model_runner_v2.yaml +++ b/.buildkite/test_areas/model_runner_v2.yaml @@ -63,7 +63,7 @@ steps: - label: ":nvidia: (L4) Model Runner V2 Distributed" key: model-runner-v2-distributed-2-gpus - timeout_in_minutes: 30 + timeout_in_minutes: 40 working_dir: "/vllm-workspace/tests" num_devices: 2 source_file_dependencies: @@ -72,6 +72,8 @@ steps: - tests/basic_correctness/test_basic_correctness.py - tests/v1/distributed/test_async_llm_dp.py - tests/v1/distributed/test_eagle_dp.py + - tests/v1/e2e/general/test_sharded_sampling.py + - tests/v1/e2e/spec_decode/test_sharded_sampling.py commands: - set -x - export VLLM_USE_V2_MODEL_RUNNER=1 @@ -81,6 +83,9 @@ steps: - export NCCL_CUMEM_HOST_ENABLE=0 - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py -k "not ray" - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py + - pytest -v -s v1/e2e/general/test_sharded_sampling.py + - pytest -v -s v1/e2e/spec_decode/test_sharded_sampling.py + mirror: amd: label: ":amd: (MI300) Model Runner V2 Distributed" diff --git a/tests/v1/e2e/general/test_sharded_sampling.py b/tests/v1/e2e/general/test_sharded_sampling.py new file mode 100644 index 0000000000..a580605fb0 --- /dev/null +++ b/tests/v1/e2e/general/test_sharded_sampling.py @@ -0,0 +1,162 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Batch-sharded sampling must be a drop-in for replicated sampling. + +The comparison is tolerance-based, not bitwise: engine boots land in +slightly different kernel/collective states (measured: same-state boots are +bitwise identical across the two modes; cross-state boots differ by up to +~0.25 logprob and can flip a near-tie token). A real sharding bug — rows +routed to the wrong request, misaligned gathers — produces many-nats +divergence and scrambled top-k sets, far outside these bounds.""" + +import pytest + +from tests.utils import multi_gpu_test +from vllm import LLM, SamplingParams +from vllm.distributed import cleanup_dist_env_and_memory + +MODEL = "meta-llama/Llama-3.2-1B-Instruct" + +# Boot-state numeric noise bound (measured <= 0.25 on identical contexts). +LOGPROB_TOL = 0.5 +# Near-tie greedy/seeded flips from boot-state noise; measured <= 1 per +# engine boot across ~16 completions, plus one of headroom. +MAX_DIVERGENT_PROMPTS = 3 + +PROMPTS = [ + "The capital of France is", + "The three primary colors are", + "Write a haiku about the ocean.", + "1, 1, 2, 3, 5, 8,", + "The chemical symbol for gold is", + "Once upon a time, in a distant kingdom,", + "Python's GIL stands for", + "The fastest land animal is", +] + +# Exercise greedy, seeded random, top-k/top-p, penalties, and logprobs. +SAMPLING_PARAMS = [ + SamplingParams(temperature=0.0, max_tokens=32, logprobs=5, prompt_logprobs=1), + SamplingParams(temperature=0.8, seed=42, top_p=0.9, max_tokens=32, logprobs=3), + SamplingParams(temperature=0.7, seed=7, top_k=50, min_p=0.05, max_tokens=32), + SamplingParams( + temperature=0.9, + seed=123, + frequency_penalty=0.5, + repetition_penalty=1.2, + max_tokens=32, + logprobs=2, + ), +] + + +def _is_sharded_sampling_active(worker) -> bool: + return worker.model_runner.batch_sharder is not None + + +def _generate(monkeypatch: pytest.MonkeyPatch, disable_sharding: bool): + llm = LLM( + model=MODEL, + tensor_parallel_size=2, + enable_batch_sharded_sampling=not disable_sharding, + max_model_len=1024, + # More requests than ranks, so round-robin slot ownership splits the + # batch across both ranks. + max_num_seqs=len(PROMPTS), + enforce_eager=True, + enable_prefix_caching=False, + gpu_memory_utilization=0.7, + # Timing-based kernel selection is the dominant source of cross-boot + # numeric noise; disable it so the comparison bounds stay tight. + kernel_config={"enable_flashinfer_autotune": False}, + ) + try: + # Guard against a vacuous comparison: verify every worker is in the + # intended sampling mode. + modes = llm.llm_engine.collective_rpc(_is_sharded_sampling_active) + assert all(mode == (not disable_sharding) for mode in modes), modes + params = [ + SAMPLING_PARAMS[i % len(SAMPLING_PARAMS)] for i in range(len(PROMPTS)) + ] + # Two waves: the second reuses the request slots freed by the first + # (in whatever order requests finished), covering the cross-rank + # slot-recycling path that request ownership derives from. + wave1 = llm.generate(PROMPTS, params) + wave2 = llm.generate(PROMPTS[::-1], params[::-1]) + return wave1 + wave2 + finally: + del llm + cleanup_dist_env_and_memory() + + +def _assert_logprob_dicts_close(ref_lps, out_lps, what: str) -> None: + # Allow one boundary entry of the top-k set to swap at a near-tie. + common = set(ref_lps) & set(out_lps) + assert len(common) >= max(len(ref_lps), len(out_lps)) - 1, ( + f"{what}: top-k token sets diverge: {sorted(ref_lps)} vs {sorted(out_lps)}" + ) + for token_id in common: + diff = abs(ref_lps[token_id].logprob - out_lps[token_id].logprob) + assert diff <= LOGPROB_TOL, ( + f"{what}[{token_id}]: logprob diff {diff} " + f"({ref_lps[token_id].logprob} vs {out_lps[token_id].logprob})" + ) + + +@multi_gpu_test(num_gpus=2) +def test_sharded_sampling_outputs_match(monkeypatch: pytest.MonkeyPatch): + """Generation, logprobs, and prompt logprobs match between batch-sharded + sampling and the replicated fallback, up to measured boot-state noise.""" + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") + # Required for the collective_rpc mode check below. + monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") + + ref_outputs = _generate(monkeypatch, disable_sharding=True) + shard_outputs = _generate(monkeypatch, disable_sharding=False) + + assert len(ref_outputs) == len(shard_outputs) == 2 * len(PROMPTS) + num_divergent = 0 + for i, (ref, out) in enumerate(zip(ref_outputs, shard_outputs)): + ref_completion = ref.outputs[0] + out_completion = out.outputs[0] + ref_ids = list(ref_completion.token_ids) + out_ids = list(out_completion.token_ids) + + # Logits are only comparable at positions whose context (prompt + + # previously sampled tokens) is identical: every position up to and + # including the first divergence. + prefix = 0 + while prefix < min(len(ref_ids), len(out_ids)): + if ref_ids[prefix] != out_ids[prefix]: + break + prefix += 1 + if ref_ids != out_ids: + num_divergent += 1 + + if ref_completion.logprobs is not None: + assert out_completion.logprobs is not None + comparable = min(prefix + 1, len(ref_ids), len(out_ids)) + for pos in range(comparable): + _assert_logprob_dicts_close( + ref_completion.logprobs[pos], + out_completion.logprobs[pos], + f"prompt {i} logprobs[{pos}]", + ) + + # The prompt is fixed, so every prompt-logprob position is comparable. + if ref.prompt_logprobs is not None: + assert out.prompt_logprobs is not None + for pos, (ref_lps, out_lps) in enumerate( + zip(ref.prompt_logprobs, out.prompt_logprobs) + ): + if ref_lps is None or out_lps is None: + assert ref_lps is None and out_lps is None + continue + _assert_logprob_dicts_close( + ref_lps, out_lps, f"prompt {i} prompt_logprobs[{pos}]" + ) + + assert num_divergent <= MAX_DIVERGENT_PROMPTS, ( + f"{num_divergent}/{2 * len(PROMPTS)} prompts diverged: beyond near-tie " + "boot-state noise; sharded sampling is likely misrouting requests" + ) diff --git a/tests/v1/e2e/spec_decode/test_sharded_sampling.py b/tests/v1/e2e/spec_decode/test_sharded_sampling.py new file mode 100644 index 0000000000..cc9781ca9f --- /dev/null +++ b/tests/v1/e2e/spec_decode/test_sharded_sampling.py @@ -0,0 +1,102 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Batch-sharded sampling must be a drop-in for replicated sampling under +speculative decoding. + +The non-spec arm of this claim lives in tests/v1/e2e/general/ +test_sharded_sampling.py. Spec decode is the harder case: draft tokens give +each request a different number of logits rows, so the shard plan's +per-request arithmetic and the rejection sampler are both exercised. +""" + +import pytest +import torch + +from tests.utils import wait_for_memory_to_settle +from vllm import LLM, SamplingParams +from vllm.distributed import cleanup_dist_env_and_memory + +from .utils import _skip_if_insufficient_gpus_for_tp, get_test_prompts, greedy_sampling + + +def _is_sharded_sampling_active(worker) -> bool: + return worker.model_runner.batch_sharder is not None + + +def test_mtp_sharded_sampling_equivalence(monkeypatch: pytest.MonkeyPatch): + """Batch-sharded sampling must be a bit-exact drop-in for replicated + sampling under MTP spec decoding: the collectives move the same logits + bytes, Gumbel keys derive from (request slot, position, seed), and slot + assignment is rank-deterministic. Both runs here are spec decode with + identical math, so outputs must match exactly.""" + tp_size = 2 + _skip_if_insufficient_gpus_for_tp(tp_size) + model_name = "Qwen/Qwen3.5-0.8B-Base" + test_prompts = get_test_prompts(mm_enabled=False, num_prompts=20) + + with monkeypatch.context() as m: + # Batch-sharded sampling only exists in the V2 model runner. + m.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") + # Required for the collective_rpc mode check below. + m.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") + + def run(disable_sharding: bool): + llm = LLM( + model=model_name, + tensor_parallel_size=tp_size, + enable_batch_sharded_sampling=not disable_sharding, + speculative_config={ + "method": "mtp", + "num_speculative_tokens": 1, + "max_model_len": 2048, + }, + max_model_len=2048, + limit_mm_per_prompt={"image": 0, "video": 0}, + # More requests than ranks, so round-robin slot ownership + # splits the batch across both ranks. + max_num_seqs=len(test_prompts), + # Timing-based kernel selection is the dominant source of + # cross-boot numeric noise; disable for a tight comparison. + kernel_config={"enable_flashinfer_autotune": False}, + ) + try: + # Guard against a vacuous comparison: verify every worker is + # in the intended sampling mode. + modes = llm.llm_engine.collective_rpc(_is_sharded_sampling_active) + assert all(mode == (not disable_sharding) for mode in modes) + greedy = llm.chat(test_prompts, greedy_sampling()) + seeded = llm.chat( + test_prompts, + SamplingParams(temperature=1.0, seed=33, max_tokens=10), + ) + return greedy, seeded + finally: + del llm + torch.accelerator.empty_cache() + cleanup_dist_env_and_memory() + wait_for_memory_to_settle() + + ref_greedy, ref_seeded = run(disable_sharding=True) + shard_greedy, shard_seeded = run(disable_sharding=False) + + # Engine boots are not bitwise deterministic (kernel/collective + # selection shifts logits by ~1 ulp, flipping near-tie tokens), so + # allow a small number of divergent prompts. A sharding bug produces + # wholesale divergence, not isolated near-tie flips. + for name, ref_outputs, shard_outputs in ( + ("greedy", ref_greedy, shard_greedy), + ("seeded", ref_seeded, shard_seeded), + ): + num_divergent = 0 + for i, (ref, out) in enumerate(zip(ref_outputs, shard_outputs)): + if list(ref.outputs[0].token_ids) != list(out.outputs[0].token_ids): + num_divergent += 1 + print( + f"{name} prompt {i} diverged:\n" + f" replicated: {ref.outputs[0].text!r}\n" + f" sharded: {out.outputs[0].text!r}" + ) + assert num_divergent <= 2, ( + f"{name}: {num_divergent}/{len(ref_outputs)} prompts diverged, " + "beyond near-tie boot noise" + ) diff --git a/tests/v1/worker/test_gpu_batch_shard.py b/tests/v1/worker/test_gpu_batch_shard.py new file mode 100644 index 0000000000..09972b9ff6 --- /dev/null +++ b/tests/v1/worker/test_gpu_batch_shard.py @@ -0,0 +1,613 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Routing-math tests for batch-sharded sampling. + +These simulate every TP rank in-process (no torch.distributed): the local +sub-batch, the all-to-all send/recv layout, and the padded result gather must +compose to a bit-exact round trip of the replicated computation. +""" + +from types import SimpleNamespace +from unittest import mock + +import numpy as np +import pytest +import torch + +from vllm.v1.core.sched.output import GrammarOutput +from vllm.v1.worker.gpu.input_batch import InputBatch +from vllm.v1.worker.gpu.sample import batch_shard +from vllm.v1.worker.gpu.sample.batch_shard import ( + BatchSharder, + _shard_grammar_output, +) + +DEVICE = "cuda" + +# The owner-sorted layout is expanded by a Triton kernel, so the routing math +# can only be exercised on device. +requires_cuda = pytest.mark.skipif( + not torch.cuda.is_available(), reason="requires CUDA" +) + + +def _np(t: torch.Tensor) -> np.ndarray: + return t.cpu().numpy() + + +def _make_batch( + rng: np.random.Generator, + num_reqs: int, + max_num_reqs: int, + max_spec: int, + slot_pool: np.ndarray | None = None, +) -> InputBatch: + """Consistent batch with random unique request slots, 1 + k_i logits + rows per request at the tail of each query segment.""" + if slot_pool is None: + slot_pool = np.arange(max_num_reqs) + slots = rng.choice(slot_pool, size=num_reqs, replace=False) + slots = slots.astype(np.int32) + k = rng.integers(0, max_spec + 1, size=num_reqs) + num_logits_per_req = (1 + k).astype(np.int32) + cu = np.zeros(num_reqs + 1, dtype=np.int32) + np.cumsum(num_logits_per_req, out=cu[1:]) + + query_lens = num_logits_per_req + rng.integers(0, 4, size=num_reqs).astype(np.int32) + qsl = np.zeros(num_reqs + 1, dtype=np.int32) + np.cumsum(query_lens, out=qsl[1:]) + num_tokens = int(qsl[-1]) + + logits_indices_np = np.concatenate( + [ + np.arange(qsl[i + 1] - num_logits_per_req[i], qsl[i + 1]) + for i in range(num_reqs) + ] + ).astype(np.int64) + expanded_idx_np = np.repeat(slots, num_logits_per_req) + expanded_pos_np = np.concatenate( + [np.arange(n, dtype=np.int32) for n in num_logits_per_req] + ) + seq_lens_np = (query_lens + rng.integers(0, 100, size=num_reqs)).astype(np.int32) + + def dev(a: np.ndarray) -> torch.Tensor: + return torch.from_numpy(a).to(DEVICE) + + return InputBatch( + req_ids=[f"req-{i}" for i in range(num_reqs)], + num_reqs=num_reqs, + num_reqs_after_padding=num_reqs, + idx_mapping=dev(slots), + idx_mapping_np=slots, + expanded_idx_mapping=dev(expanded_idx_np), + expanded_local_pos=dev(expanded_pos_np), + num_scheduled_tokens=query_lens, + num_tokens=num_tokens, + num_tokens_after_padding=num_tokens, + num_draft_tokens=int(k.sum()), + num_draft_tokens_per_req=k.astype(np.int32), + query_start_loc=dev(qsl), + query_start_loc_np=qsl, + seq_lens=dev(seq_lens_np), + seq_lens_cpu_upper_bound=torch.from_numpy(seq_lens_np.copy()), + dcp_local_seq_lens=None, + num_computed_tokens_np=np.zeros(num_reqs, dtype=np.int32), + prefill_len_np=np.zeros(num_reqs, dtype=np.int32), + num_computed_prefill_tokens_np=np.zeros(num_reqs, dtype=np.int32), + is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_), + has_prefill=False, + max_seq_len_np=None, + input_ids=torch.zeros(num_tokens, dtype=torch.int32, device=DEVICE), + positions=torch.arange(num_tokens, dtype=torch.int64, device=DEVICE), + is_padding=torch.zeros(num_tokens, dtype=torch.bool, device=DEVICE), + logits_indices=dev(logits_indices_np), + cu_num_logits=dev(cu), + cu_num_logits_np=cu, + has_structured_output_reqs=False, + prompt_lens=None, + ) + + +def _shard_all_ranks(batch: InputBatch, max_num_reqs: int, tp_size: int): + """Simulate every TP rank by faking get_tp_group() per BatchSharder.""" + results = [] + for rank in range(tp_size): + fake_group = SimpleNamespace(rank_in_group=rank, world_size=tp_size) + with mock.patch.object(batch_shard, "get_tp_group", return_value=fake_group): + sharder = BatchSharder( + max_num_reqs=max_num_reqs, + max_num_logits_per_req=8, + device=torch.device(DEVICE), + ) + local_batch, sorted_logits_indices, _, metadata = sharder.shard_sampler_inputs( + batch, None + ) + results.append((local_batch, sorted_logits_indices, metadata)) + return results + + +def _owned_batch_indices(local_batch: InputBatch) -> np.ndarray: + return np.array([int(r.split("-")[1]) for r in local_batch.req_ids], dtype=np.int64) + + +def _owned_rows(cu: np.ndarray, owned: np.ndarray) -> np.ndarray: + if len(owned) == 0: + return np.array([], dtype=np.int64) + return np.concatenate([np.arange(cu[r], cu[r + 1]) for r in owned]) + + +@requires_cuda +@pytest.mark.parametrize("tp_size", [2, 4, 8]) +@pytest.mark.parametrize("seed", [0, 1]) +def test_local_batch_partition(tp_size: int, seed: int): + """Local sub-batches partition the batch by owner, preserve batch order + within an owner, and carry exactly the owned requests' entries of every + per-request and per-row array.""" + rng = np.random.default_rng(seed) + max_num_reqs = 64 + batch = _make_batch(rng, num_reqs=23, max_num_reqs=max_num_reqs, max_spec=3) + num_logits = int(batch.cu_num_logits_np[-1]) + results = _shard_all_ranks(batch, max_num_reqs, tp_size) + cu = batch.cu_num_logits_np + + metadata0 = results[0][2] + assert sum(metadata0.num_logits_per_rank) == num_logits + # Gather width = max logits rows of any request (bounds num_sampled). + assert metadata0.max_num_logits_per_req == int(np.diff(cu).max()) + + all_owned: list[int] = [] + for rank, (local, _, metadata) in enumerate(results): + owned = _owned_batch_indices(local) + assert local.num_reqs == len(owned) == metadata.num_local_reqs + assert (batch.idx_mapping_np[owned] % tp_size == rank).all() + # Stable sort: batch order is preserved within an owner. + assert (np.diff(owned) > 0).all() + all_owned.extend(owned.tolist()) + + rows = _owned_rows(cu, owned) + assert ( + metadata.num_local_logits + == len(rows) + == metadata0.num_logits_per_rank[rank] + ) + + expected_idx = batch.idx_mapping_np[owned] + assert (local.idx_mapping_np == expected_idx).all() + assert _np(local.idx_mapping).tolist() == expected_idx.tolist() + assert (np.diff(local.cu_num_logits_np) == np.diff(cu)[owned]).all() + assert _np(local.cu_num_logits).tolist() == local.cu_num_logits_np.tolist() + assert ( + _np(local.expanded_idx_mapping) == _np(batch.expanded_idx_mapping)[rows] + ).all() + assert ( + _np(local.expanded_local_pos) == _np(batch.expanded_local_pos)[rows] + ).all() + assert (_np(local.logits_indices) == _np(batch.logits_indices)[rows]).all() + assert (_np(local.seq_lens) == _np(batch.seq_lens)[owned]).all() + assert batch.num_draft_tokens_per_req is not None + assert local.num_draft_tokens_per_req is not None + assert ( + local.num_draft_tokens_per_req == batch.num_draft_tokens_per_req[owned] + ).all() + assert local.num_draft_tokens == int( + batch.num_draft_tokens_per_req[owned].sum() + ) + # Triton specializes kernels on pointer alignment: every packed-buffer + # view fed to kernels must stay 16-byte aligned across batch shapes, + # or serving JITs new kernel variants mid-collective. + for t in ( + local.idx_mapping, + local.expanded_idx_mapping, + local.expanded_local_pos, + local.logits_indices, + local.cu_num_logits, + metadata.gathered_src_indices, + ): + assert t.data_ptr() % 16 == 0 + assert sorted(all_owned) == list(range(batch.num_reqs)) + + # sorted_logits_indices is identical on all ranks: the owner-sorted logits row + # positions, i.e. logits_indices permuted rank-major. + sorted_logits_indices = _np(results[0][1]) + for _, gi, _ in results: + assert (_np(gi) == sorted_logits_indices).all() + rows_all = np.concatenate( + [_owned_rows(cu, _owned_batch_indices(local)) for local, _, _ in results] + ) + assert ( + sorted_logits_indices.tolist() == _np(batch.logits_indices)[rows_all].tolist() + ) + + +@requires_cuda +@pytest.mark.parametrize("tp_size", [2, 4]) +def test_all_to_all_reassembly_simulated(tp_size: int): + """Vocab shards computed from owner-sorted hidden states and routed + through the simulated all-to-all reassemble into exactly the full-vocab + logits of each rank's owned rows.""" + rng = np.random.default_rng(7) + max_num_reqs = 32 + batch = _make_batch(rng, num_reqs=11, max_num_reqs=max_num_reqs, max_spec=4) + vocab = 64 * tp_size + shard_w = vocab // tp_size + token_logits = torch.randn(batch.num_tokens, vocab, device=DEVICE) + + results = _shard_all_ranks(batch, max_num_reqs, tp_size) + sorted_logits_indices = results[0][1] + num_logits_per_rank = results[0][2].num_logits_per_rank + bounds = np.zeros(tp_size + 1, dtype=np.int64) + np.cumsum(num_logits_per_rank, out=bounds[1:]) + + # send_r = rank r's vocab shard of the owner-sorted rows, with no further + # permute; chunk d of send_r goes to rank d. + sends = [ + token_logits[sorted_logits_indices][:, r * shard_w : (r + 1) * shard_w] + for r in range(tp_size) + ] + for rank, (local, _, metadata) in enumerate(results): + recv = torch.cat( + [sends[src][bounds[rank] : bounds[rank + 1]] for src in range(tp_size)] + ) + # Reassemble source-rank-major chunks into full-vocab rows, mirroring + # all_to_all_logits (source-rank order is vocab-shard order). + reassembled = ( + recv.view(tp_size, metadata.num_local_logits, shard_w) + .permute(1, 0, 2) + .reshape(metadata.num_local_logits, vocab) + ) + expected = token_logits[local.logits_indices] + assert torch.equal(reassembled, expected) + + +@requires_cuda +def test_shard_no_spec(): + """Batches without draft tokens (one logits row per request) degenerate + cleanly: rows mirror requests.""" + rng = np.random.default_rng(11) + tp_size = 4 + max_num_reqs = 32 + batch = _make_batch(rng, num_reqs=10, max_num_reqs=max_num_reqs, max_spec=0) + results = _shard_all_ranks(batch, max_num_reqs, tp_size) + + assert results[0][2].max_num_logits_per_req == 1 + for local, _, metadata in results: + assert metadata.num_local_logits == metadata.num_local_reqs == local.num_reqs + owned = _owned_batch_indices(local) + assert _np(local.logits_indices).tolist() == ( + _np(batch.logits_indices)[owned].tolist() + ) + + +@pytest.mark.parametrize("tp_size", [2, 4]) +def test_shard_grammar_output(tp_size: int): + """The grammar filter keeps exactly the owned requests' bitmask rows, in + grammar order, and the per-rank pieces partition the global bitmask.""" + rng = np.random.default_rng(5) + max_num_reqs = 32 + num_reqs = 12 + slots = rng.choice(np.arange(max_num_reqs), size=num_reqs, replace=False) + k = rng.integers(0, 4, size=num_reqs) + cu = np.zeros(num_reqs + 1, dtype=np.int32) + np.cumsum(1 + k, out=cu[1:]) + owner = slots % tp_size + + # Grammar requests: every other request, in a scrambled order. + req_ids = [f"req-{i}" for i in range(num_reqs)] + grammar_idx = list(rng.permutation(np.arange(0, num_reqs, 2))) + grammar_req_ids = [req_ids[i] for i in grammar_idx] + num_grammar_rows = sum(int(cu[i + 1] - cu[i]) for i in grammar_idx) + bitmask = rng.integers(0, 2**31, size=(num_grammar_rows, 4), dtype=np.int32) + input_batch = SimpleNamespace(req_ids=req_ids, cu_num_logits_np=cu) + + # Global row offset of each grammar request within the bitmask. + row_offsets = {} + cursor = 0 + for i in grammar_idx: + row_offsets[i] = cursor + cursor += int(cu[i + 1] - cu[i]) + + grammar_output = GrammarOutput( + structured_output_request_ids=grammar_req_ids, + grammar_bitmask=bitmask, + ) + + kept_total = 0 + for rank in range(tp_size): + local_req_ids = [req_ids[i] for i in range(num_reqs) if owner[i] == rank] + local = _shard_grammar_output(grammar_output, input_batch, local_req_ids) + expected_idx = [i for i in grammar_idx if owner[i] == rank] + if not expected_idx: + assert local is None + continue + assert local is not None + assert local.structured_output_request_ids == [req_ids[i] for i in expected_idx] + expected_rows = np.concatenate( + [ + np.arange(row_offsets[i], row_offsets[i] + cu[i + 1] - cu[i]) + for i in expected_idx + ] + ) + assert (local.grammar_bitmask == bitmask[expected_rows.astype(int)]).all() + kept_total += local.grammar_bitmask.shape[0] + assert kept_total == num_grammar_rows + + +@requires_cuda +@pytest.mark.parametrize("tp_size", [2, 4]) +def test_gather_src_indices_round_trip(tp_size: int): + """Padded per-rank result blocks indexed by gathered_src_indices restore + batch order, including when some rank owns zero requests.""" + rng = np.random.default_rng(3) + max_num_reqs = 40 + all_slots = np.arange(max_num_reqs) + batch = _make_batch( + rng, + num_reqs=17, + max_num_reqs=max_num_reqs, + max_spec=2, + slot_pool=all_slots[all_slots % tp_size != 0], # rank 0 owns nothing + ) + results = _shard_all_ranks(batch, max_num_reqs, tp_size) + metadata0 = results[0][2] + assert metadata0.num_local_reqs == 0 + + # Value of request j is j; each rank packs its owned requests' values into + # a zero-padded (max_num_reqs_per_rank, 1) block, gathered rank-major. + pad = metadata0.max_num_reqs_per_rank + blocks = [] + for local, _, _ in results: + owned = _owned_batch_indices(local) + block = torch.zeros(pad, 1, dtype=torch.int64, device=DEVICE) + block[: len(owned), 0] = torch.from_numpy(owned).to(DEVICE) + blocks.append(block) + gathered = torch.cat(blocks) + + restored = gathered[metadata0.gathered_src_indices][:, 0] + assert restored.tolist() == list(range(batch.num_reqs)) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +@pytest.mark.parametrize("tp_size", [2, 4]) +def test_gather_sampler_output_kernels(tp_size: int): + """The fused pack/unpack kernels round-trip per-rank sampler results + into batch order: token ids zero-filled past the local width, counts + cast back to int32, identical on every rank.""" + from dataclasses import replace as dc_replace + + from vllm.v1.worker.gpu.sample.output import SamplerOutput + + rng = np.random.default_rng(9) + max_num_reqs = 32 + batch = _make_batch(rng, num_reqs=13, max_num_reqs=max_num_reqs, max_spec=3) + results = _shard_all_ranks(batch, max_num_reqs, tp_size) + device = torch.device("cuda") + num_reqs = batch.num_reqs + width = results[0][2].max_num_logits_per_req + + # Distinctive per-rank local outputs. Even ranks mimic the regular + # sampler (one column, exercising zero-fill); odd ranks mimic the + # rejection sampler's over-allocation (exercising truncation). + local_outputs: list[SamplerOutput | None] = [] + expected_ids = torch.zeros(num_reqs, width, dtype=torch.int64) + expected_sampled = torch.zeros(num_reqs, dtype=torch.int32) + expected_rejected = torch.zeros(num_reqs, dtype=torch.int32) + for rank, (local, _, metadata) in enumerate(results): + if metadata.num_local_reqs == 0: + local_outputs.append(None) + continue + owned = torch.from_numpy(_owned_batch_indices(local)) + src_width = 1 if rank % 2 == 0 else width + 3 + ids = owned[:, None] * 1000 + torch.arange(src_width)[None, :] + num_sampled = (owned % width + 1).to(torch.int32) + num_rejected = (owned % 3).to(torch.int32) + local_outputs.append( + SamplerOutput( + sampled_token_ids=ids.to(device), + logprobs_tensors=None, + num_nans=None, + num_sampled=num_sampled.to(device), + num_rejected=num_rejected.to(device), + ) + ) + copy_width = min(src_width, width) + expected_ids[owned, :copy_width] = ids[:, :copy_width] + expected_sampled[owned] = num_sampled + expected_rejected[owned] = num_rejected + + # Two passes over a mocked all-gather: capture each rank's packed send + # block, then replay with the concatenated blocks as the gathered result. + packed_blocks: dict[int, torch.Tensor] = {} + gathered_full: list[torch.Tensor] = [] + + def run(rank: int) -> SamplerOutput: + metadata = dc_replace( + results[rank][2], + gathered_src_indices=results[rank][2].gathered_src_indices.to(device), + ) + + def fake_all_gather(x: torch.Tensor, dim: int = 0) -> torch.Tensor: + packed_blocks[rank] = x.clone() + if gathered_full: + return gathered_full[0] + return torch.zeros( + tp_size * x.shape[0], x.shape[1], dtype=x.dtype, device=x.device + ) + + with mock.patch.object( + batch_shard, "tensor_model_parallel_all_gather", fake_all_gather + ): + return batch_shard.gather_sampler_output( + local_outputs[rank], + metadata, + device, + global_batch=batch, + local_batch=results[rank][0], + ) + + for rank in range(tp_size): + run(rank) + # Zero the uninitialized padding rows so the comparison is deterministic; + # the real path never reads them (gathered_src_indices skips padding). + for rank, (local, _, metadata) in enumerate(results): + packed_blocks[rank][metadata.num_local_reqs :] = 0 + gathered_full.append(torch.cat([packed_blocks[r] for r in range(tp_size)])) + + for rank in range(tp_size): + out = run(rank) + assert out.sampled_token_ids.dtype == torch.int64 + assert out.num_sampled is not None and out.num_sampled.dtype == torch.int32 + assert out.num_rejected is not None + assert torch.equal(out.sampled_token_ids.cpu(), expected_ids) + assert torch.equal(out.num_sampled.cpu(), expected_sampled) + assert torch.equal(out.num_rejected.cpu(), expected_rejected.to(torch.int32)) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_gather_sampler_output_logprobs_and_nans(): + """num_nans is reduced per request and gathered; LogprobsTensors are + packed into fixed per-request blocks, width-aligned across ranks (pad + and truncate paths), and restored to global row order.""" + from dataclasses import replace as dc_replace + + from vllm.v1.outputs import LogprobsTensors + from vllm.v1.worker.gpu.sample.output import SamplerOutput + + tp_size = 2 + rng = np.random.default_rng(21) + max_num_reqs = 32 + batch = _make_batch(rng, num_reqs=6, max_num_reqs=max_num_reqs, max_spec=3) + results = _shard_all_ranks(batch, max_num_reqs, tp_size) + device = torch.device("cuda") + num_reqs = batch.num_reqs + cu = batch.cu_num_logits_np + num_logits = int(cu[-1]) + width = results[0][2].max_num_logits_per_req + num_cols = 3 + + def cuda_fields(b: InputBatch) -> InputBatch: + return dc_replace( + b, + cu_num_logits=b.cu_num_logits.to(device), + expanded_local_pos=b.expanded_local_pos.to(device), + ) + + global_cuda = cuda_fields(batch) + + local_outputs: list[SamplerOutput | None] = [] + local_batches: list[InputBatch] = [] + expected_ids = torch.zeros(num_logits, num_cols, dtype=torch.int64) + expected_logprobs = torch.full( + (num_logits, num_cols), float("-inf"), dtype=torch.float32 + ) + expected_ranks = torch.zeros(num_logits, dtype=torch.int64) + expected_nans = torch.zeros(num_reqs, dtype=torch.int32) + for rank, (local, _, metadata) in enumerate(results): + local_batches.append(cuda_fields(local)) + if metadata.num_local_reqs == 0: + local_outputs.append(None) + continue + owned = _owned_batch_indices(local) + rows = torch.from_numpy(_owned_rows(cu, owned)) + # Rank 0 sends fewer columns (pad path), rank 1 more (truncate path). + local_cols = num_cols - 1 if rank == 0 else num_cols + 2 + ids = rows[:, None] * 10 + torch.arange(local_cols)[None, :] + logprobs = ids.float() * 0.5 + ranks_t = rows + 7 + nans = (rows % 4).to(torch.int32) + local_outputs.append( + SamplerOutput( + sampled_token_ids=torch.zeros( + metadata.num_local_reqs, width, dtype=torch.int64, device=device + ), + logprobs_tensors=LogprobsTensors( + logprob_token_ids=ids.to(device), + logprobs=logprobs.to(device), + selected_token_ranks=ranks_t.to(device), + cu_num_generated_tokens=None, + ), + num_nans=nans.to(device), + num_sampled=torch.ones( + metadata.num_local_reqs, dtype=torch.int32, device=device + ), + num_rejected=torch.zeros( + metadata.num_local_reqs, dtype=torch.int32, device=device + ), + ) + ) + copy_cols = min(local_cols, num_cols) + expected_ids[rows, :copy_cols] = ids[:, :copy_cols] + expected_logprobs[rows, :copy_cols] = logprobs[:, :copy_cols] + expected_ranks[rows] = ranks_t + for j in owned: + in_req = (rows >= int(cu[j])) & (rows < int(cu[j + 1])) + expected_nans[j] = int(nans[in_req].sum()) + + recorded: dict[int, list[torch.Tensor]] = {} + gathered_full: list[torch.Tensor] = [] + + def run(rank: int) -> SamplerOutput: + metadata = dc_replace( + results[rank][2], + gathered_src_indices=results[rank][2].gathered_src_indices.to(device), + ) + calls = {"i": 0} + + def fake_all_gather(x: torch.Tensor, dim: int = 0) -> torch.Tensor: + i = calls["i"] + calls["i"] += 1 + if gathered_full: + return gathered_full[i] + recorded.setdefault(rank, []).append(x.clone()) + return torch.zeros( + tp_size * x.shape[0], *x.shape[1:], dtype=x.dtype, device=x.device + ) + + with mock.patch.object( + batch_shard, "tensor_model_parallel_all_gather", fake_all_gather + ): + return batch_shard.gather_sampler_output( + local_outputs[rank], + metadata, + device, + global_batch=global_cuda, + local_batch=local_batches[rank], + gather_num_nans=True, + logprobs_dims=(num_cols - 1, 0), + ) + + for rank in range(tp_size): + run(rank) + gathered_full.extend( + torch.cat([recorded[r][i] for r in range(tp_size)]) for i in range(3) + ) + + for rank in range(tp_size): + out = run(rank) + assert out.num_nans is not None + assert torch.equal(out.num_nans.cpu(), expected_nans) + lp = out.logprobs_tensors + assert lp is not None + assert torch.equal(lp.logprob_token_ids.cpu(), expected_ids) + assert torch.equal(lp.logprobs.cpu(), expected_logprobs) + assert torch.equal(lp.selected_token_ranks.cpu(), expected_ranks) + if num_logits != num_reqs: + assert lp.cu_num_generated_tokens == cu.tolist() + else: + assert lp.cu_num_generated_tokens is None + + +def test_finish_requests_frees_slots_in_sorted_order(): + """Request slots must be freed in the same order on every TP rank: + ownership derives from slot indices, and `finished_req_ids` is a set + whose iteration order is per-process hash-randomized.""" + from vllm.v1.worker.gpu.model_runner import GPUModelRunner + + removed: list[str] = [] + fake_runner = SimpleNamespace(_remove_request=removed.append, pooling_runner=None) + scheduler_output = SimpleNamespace( + finished_req_ids={"req-b", "req-c", "req-a"}, + preempted_req_ids={"req-e", "req-d"}, + ) + GPUModelRunner.finish_requests(fake_runner, scheduler_output) + assert removed == sorted(removed) + assert set(removed) == {"req-a", "req-b", "req-c", "req-d", "req-e"} diff --git a/tests/v1/worker/test_gpu_model_runner_v2_eplb.py b/tests/v1/worker/test_gpu_model_runner_v2_eplb.py index 637d078c73..6a4603db38 100644 --- a/tests/v1/worker/test_gpu_model_runner_v2_eplb.py +++ b/tests/v1/worker/test_gpu_model_runner_v2_eplb.py @@ -60,6 +60,7 @@ def _make_runner(**overrides: Any) -> Any: runner.parallel_config = SimpleNamespace( enable_eplb=True, enable_elastic_ep=False, + enable_batch_sharded_sampling=False, eplb_config=SimpleNamespace(log_balancedness=True), ) runner.vllm_config = SimpleNamespace( diff --git a/vllm/config/parallel.py b/vllm/config/parallel.py index 15c0116e4d..0a10641b9e 100644 --- a/vllm/config/parallel.py +++ b/vllm/config/parallel.py @@ -164,6 +164,13 @@ class ParallelConfig: """Whether the deployed model is MoE (if known).""" enable_expert_parallel: bool = False """Use expert parallelism instead of tensor parallelism for MoE layers.""" + enable_batch_sharded_sampling: bool | None = None + """Use sharded sampling across tensor parallel ranks. Each rank samples + a slice of the batch instead of every rank sampling all of it. Currently + defaults to False if not set. Enabling it explicitly raises when the config + cannot support it (`tensor_parallel_size` must be > 1, `max_num_seqs` at + least `tensor_parallel_size`, and `max_logprobs` non-negative). Models opt in + by implementing `compute_logits_local`.""" enable_ep_weight_filter: bool = False """Skip non-local expert weights during model loading when expert parallelism is active. Each rank only reads its own expert shard from diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index acde48a931..56fa327a2b 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -1675,6 +1675,8 @@ class VllmConfig: "Remove VLLM_USE_V2_MODEL_RUNNER=0." ) + self._validate_batch_sharded_sampling() + # Re-compute compile ranges after platform-specific config updates # (e.g., XPU may lower max_num_batched_tokens when MLA is enabled) self._set_compile_ranges() @@ -2552,6 +2554,57 @@ class VllmConfig: return unsupported + def _validate_batch_sharded_sampling(self) -> None: + """Validate `enable_batch_sharded_sampling` against the rest of the config.""" + if not self.parallel_config.enable_batch_sharded_sampling: + # Default to False if not set. + self.parallel_config.enable_batch_sharded_sampling = False + return + + blockers: list[str] = [] + tp_size = self.parallel_config.tensor_parallel_size + + if not self.use_v2_model_runner: + blockers.append("it is only implemented by Model Runner V2") + + if tp_size <= 1: + blockers.append("tensor_parallel_size is 1, so there is nothing to shard") + elif self.scheduler_config.max_num_seqs < tp_size: + # Requests are assigned to ranks whole, so fewer slots than ranks + # leaves some ranks without work in every step. + blockers.append( + f"max_num_seqs ({self.scheduler_config.max_num_seqs}) is below " + f"tensor_parallel_size ({tp_size})" + ) + + if self.model_config is not None and self.model_config.max_logprobs < 0: + # max_logprobs == -1 allows vocab-size logprob requests, which the + # fixed-width logprobs gather cannot reasonably size for. + blockers.append("max_logprobs is -1, allowing vocab-size logprob requests") + + if ( + self.speculative_config is not None + and self.speculative_config.enable_adaptive_verification + ): + # Adaptive verification picks the per-request draft split on the GPU, + # so cu_num_logits_np is only an upper bound, while the shard plan is + # built from that CPU array. The two disagree once the budget binds. + # TODO(TheEpicDolphin): Support adaptive verification with batch-sharded + # sampling. + blockers.append( + "it does not yet work with adaptive verification, which decides " + "the per-request logits counts on the GPU, where the CPU-side " + "shard plan cannot see them" + ) + + if blockers: + raise ValueError( + "Batch-sharded sampling was explicitly enabled via " + "the --enable-batch-sharded-sampling flag, but is not supported " + "in this configuration for the following reason(s): " + f"{'; '.join(blockers)}." + ) + def _validate_v2_model_runner(self) -> None: """Check for features not yet supported by the V2 model runner.""" if not HAS_TRITON: diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 6c574d3238..d117fedf37 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -496,6 +496,9 @@ class EngineArgs: data_parallel_multi_port_external_lb: bool = False data_parallel_backend: DataParallelBackend = ParallelConfig.data_parallel_backend enable_expert_parallel: bool = ParallelConfig.enable_expert_parallel + enable_batch_sharded_sampling: bool | None = ( + ParallelConfig.enable_batch_sharded_sampling + ) enable_ep_weight_filter: bool = ParallelConfig.enable_ep_weight_filter moe_backend: MoEBackend = KernelConfig.moe_backend linear_backend: LinearBackend = KernelConfig.linear_backend @@ -1156,6 +1159,10 @@ class EngineArgs: "-ep", **parallel_kwargs["enable_expert_parallel"], ) + parallel_group.add_argument( + "--enable-batch-sharded-sampling", + **parallel_kwargs["enable_batch_sharded_sampling"], + ) parallel_group.add_argument( "--enable-ep-weight-filter", **parallel_kwargs["enable_ep_weight_filter"], @@ -2279,6 +2286,7 @@ class EngineArgs: data_parallel_hybrid_lb=self.data_parallel_hybrid_lb, is_moe_model=model_config.is_moe, enable_expert_parallel=self.enable_expert_parallel, + enable_batch_sharded_sampling=self.enable_batch_sharded_sampling, enable_ep_weight_filter=self.enable_ep_weight_filter, all2all_backend=self.all2all_backend, enable_elastic_ep=self.enable_elastic_ep, diff --git a/vllm/lora/layers/logits_processor.py b/vllm/lora/layers/logits_processor.py index 03f1b7ffdd..7db5462d8e 100644 --- a/vllm/lora/layers/logits_processor.py +++ b/vllm/lora/layers/logits_processor.py @@ -155,7 +155,14 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA): hidden_states: torch.Tensor, lm_head: VocabParallelEmbedding, embedding_bias: torch.Tensor | None = None, + skip_gather: bool = False, ) -> torch.Tensor | None: + # The LoRA delta is accumulated into the full gathered logits, so the + # TP gather cannot be skipped here. + if skip_gather: + raise NotImplementedError( + "Skipping the logits TP gather is not supported with an lm_head LoRA." + ) # Get the logits for the next tokens. if hasattr(lm_head, "base_layer"): actual_lm_head = lm_head.base_layer diff --git a/vllm/model_executor/layers/logits_processor.py b/vllm/model_executor/layers/logits_processor.py index e7fd568b41..c5a52cfd3a 100644 --- a/vllm/model_executor/layers/logits_processor.py +++ b/vllm/model_executor/layers/logits_processor.py @@ -99,12 +99,15 @@ class LogitsProcessor(PluggableLayer): lm_head: VocabParallelEmbedding, hidden_states: torch.Tensor, embedding_bias: torch.Tensor | None = None, + skip_gather: bool = False, ) -> torch.Tensor | None: if self.logits_as_input: logits = hidden_states else: # Get the logits for the next tokens. - logits = self._get_logits(hidden_states, lm_head, embedding_bias) + logits = self._get_logits( + hidden_states, lm_head, embedding_bias, skip_gather + ) if logits is not None: if self.soft_cap is not None: logits = logits / self.soft_cap @@ -173,9 +176,12 @@ class LogitsProcessor(PluggableLayer): hidden_states: torch.Tensor, lm_head: VocabParallelEmbedding, embedding_bias: torch.Tensor | None, + skip_gather: bool = False, ) -> torch.Tensor | None: # Get the logits for the next tokens. logits = self._apply_head(lm_head, hidden_states, embedding_bias) + if skip_gather: + return logits # Gather logits for TP if lm_head.tp_size > 1: diff --git a/vllm/model_executor/models/llama.py b/vllm/model_executor/models/llama.py index f136cea23b..d627ec76ca 100644 --- a/vllm/model_executor/models/llama.py +++ b/vllm/model_executor/models/llama.py @@ -532,6 +532,12 @@ class LlamaForCausalLM( logits = self.logits_processor(self.lm_head, hidden_states) return logits + def compute_logits_local( + self, + hidden_states: torch.Tensor, + ) -> torch.Tensor: + return self.logits_processor(self.lm_head, hidden_states, skip_gather=True) + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: loader = AutoWeightsLoader(self) return loader.load_weights(weights) diff --git a/vllm/model_executor/models/qwen3_5.py b/vllm/model_executor/models/qwen3_5.py index 2624102192..f991f05d74 100644 --- a/vllm/model_executor/models/qwen3_5.py +++ b/vllm/model_executor/models/qwen3_5.py @@ -429,6 +429,12 @@ class Qwen3_5ForCausalLMBase( ) -> torch.Tensor | None: return self.logits_processor(self.lm_head, hidden_states) + def compute_logits_local( + self, + hidden_states: torch.Tensor, + ) -> torch.Tensor: + return self.logits_processor(self.lm_head, hidden_states, skip_gather=True) + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: loader = AutoWeightsLoader(self) return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) @@ -591,6 +597,9 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid) return hidden_states + def compute_logits_local(self, hidden_states: torch.Tensor) -> torch.Tensor: + return self.language_model.compute_logits_local(hidden_states) + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: loader = AutoWeightsLoader(self) return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) diff --git a/vllm/models/deepseek_v4/nvidia/model.py b/vllm/models/deepseek_v4/nvidia/model.py index 37a43e7af7..095d8bc741 100644 --- a/vllm/models/deepseek_v4/nvidia/model.py +++ b/vllm/models/deepseek_v4/nvidia/model.py @@ -1765,6 +1765,12 @@ class DeepseekV4ForCausalLM( logits = self.logits_processor(self.lm_head, hidden_states) return logits + def compute_logits_local( + self, + hidden_states: torch.Tensor, + ) -> torch.Tensor: + return self.logits_processor(self.lm_head, hidden_states, skip_gather=True) + def forward( self, input_ids: torch.Tensor, diff --git a/vllm/models/inkling/amd/logits_processor.py b/vllm/models/inkling/amd/logits_processor.py index 2e142c5961..9cf1c663c8 100644 --- a/vllm/models/inkling/amd/logits_processor.py +++ b/vllm/models/inkling/amd/logits_processor.py @@ -63,6 +63,7 @@ class InklingLogitsProcessor(LogitsProcessor): lm_head: VocabParallelEmbedding, hidden_states: torch.Tensor, embedding_bias: torch.Tensor | None = None, + skip_gather: bool = False, ) -> torch.Tensor | None: # ``base_layer`` exists only on the LogitsProcessorWithLoRA wrapper, # which calls this forward with ``self`` bound to the wrapper. The @@ -72,15 +73,16 @@ class InklingLogitsProcessor(LogitsProcessor): # lives on this class). if hasattr(self, "base_layer"): return type(self.base_layer)._lora_forward( - self, lm_head, hidden_states, embedding_bias + self, lm_head, hidden_states, embedding_bias, skip_gather ) - return self._base_forward(lm_head, hidden_states, embedding_bias) + return self._base_forward(lm_head, hidden_states, embedding_bias, skip_gather) def _lora_forward( self, lm_head: VocabParallelEmbedding, hidden_states: torch.Tensor, embedding_bias: torch.Tensor | None = None, + skip_gather: bool = False, ) -> torch.Tensor | None: # ``self`` is the LogitsProcessorWithLoRA wrapper here: ``_get_logits`` # returns the base logits plus the lm_head LoRA delta. Apply the muP @@ -90,7 +92,9 @@ class InklingLogitsProcessor(LogitsProcessor): if self.logits_as_input: logits = hidden_states else: - logits = self._get_logits(hidden_states, lm_head, embedding_bias) + logits = self._get_logits( + hidden_states, lm_head, embedding_bias, skip_gather + ) # TODO: fuse this multiplication if logits is not None and mup: assert self.base_layer.soft_cap is None @@ -103,10 +107,11 @@ class InklingLogitsProcessor(LogitsProcessor): lm_head: VocabParallelEmbedding, hidden_states: torch.Tensor, embedding_bias: torch.Tensor | None = None, + skip_gather: bool = False, ) -> torch.Tensor | None: mup = self.logits_mup_width_multiplier if not mup: - return super().forward(lm_head, hidden_states, embedding_bias) + return super().forward(lm_head, hidden_states, embedding_bias, skip_gather) # Fold the muP width divisor into the lm_head GEMM alpha (fp32 epilogue): # no separate elementwise kernel, no bf16 rounding of scaled logits, and # no weight mutation. Overfit to the served checkpoint: bf16 lm_head, no @@ -123,6 +128,8 @@ class InklingLogitsProcessor(LogitsProcessor): beta=0.0, alpha=1.0 / mup, ) + if skip_gather: + return logits logits = self._gather_logits(logits) if logits is not None: logits = logits[..., : self.org_vocab_size] diff --git a/vllm/models/inkling/nvidia/logits_processor.py b/vllm/models/inkling/nvidia/logits_processor.py index 2e142c5961..9cf1c663c8 100644 --- a/vllm/models/inkling/nvidia/logits_processor.py +++ b/vllm/models/inkling/nvidia/logits_processor.py @@ -63,6 +63,7 @@ class InklingLogitsProcessor(LogitsProcessor): lm_head: VocabParallelEmbedding, hidden_states: torch.Tensor, embedding_bias: torch.Tensor | None = None, + skip_gather: bool = False, ) -> torch.Tensor | None: # ``base_layer`` exists only on the LogitsProcessorWithLoRA wrapper, # which calls this forward with ``self`` bound to the wrapper. The @@ -72,15 +73,16 @@ class InklingLogitsProcessor(LogitsProcessor): # lives on this class). if hasattr(self, "base_layer"): return type(self.base_layer)._lora_forward( - self, lm_head, hidden_states, embedding_bias + self, lm_head, hidden_states, embedding_bias, skip_gather ) - return self._base_forward(lm_head, hidden_states, embedding_bias) + return self._base_forward(lm_head, hidden_states, embedding_bias, skip_gather) def _lora_forward( self, lm_head: VocabParallelEmbedding, hidden_states: torch.Tensor, embedding_bias: torch.Tensor | None = None, + skip_gather: bool = False, ) -> torch.Tensor | None: # ``self`` is the LogitsProcessorWithLoRA wrapper here: ``_get_logits`` # returns the base logits plus the lm_head LoRA delta. Apply the muP @@ -90,7 +92,9 @@ class InklingLogitsProcessor(LogitsProcessor): if self.logits_as_input: logits = hidden_states else: - logits = self._get_logits(hidden_states, lm_head, embedding_bias) + logits = self._get_logits( + hidden_states, lm_head, embedding_bias, skip_gather + ) # TODO: fuse this multiplication if logits is not None and mup: assert self.base_layer.soft_cap is None @@ -103,10 +107,11 @@ class InklingLogitsProcessor(LogitsProcessor): lm_head: VocabParallelEmbedding, hidden_states: torch.Tensor, embedding_bias: torch.Tensor | None = None, + skip_gather: bool = False, ) -> torch.Tensor | None: mup = self.logits_mup_width_multiplier if not mup: - return super().forward(lm_head, hidden_states, embedding_bias) + return super().forward(lm_head, hidden_states, embedding_bias, skip_gather) # Fold the muP width divisor into the lm_head GEMM alpha (fp32 epilogue): # no separate elementwise kernel, no bf16 rounding of scaled logits, and # no weight mutation. Overfit to the served checkpoint: bf16 lm_head, no @@ -123,6 +128,8 @@ class InklingLogitsProcessor(LogitsProcessor): beta=0.0, alpha=1.0 / mup, ) + if skip_gather: + return logits logits = self._gather_logits(logits) if logits is not None: logits = logits[..., : self.org_vocab_size] diff --git a/vllm/models/minimax_m3/nvidia/model.py b/vllm/models/minimax_m3/nvidia/model.py index 99e6d1f7ae..970b06d407 100644 --- a/vllm/models/minimax_m3/nvidia/model.py +++ b/vllm/models/minimax_m3/nvidia/model.py @@ -1059,6 +1059,9 @@ class MiniMaxM3SparseForCausalLM(nn.Module, SupportsPP, SupportsEagle3): def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None: return self.logits_processor(self.lm_head, hidden_states) + def compute_logits_local(self, hidden_states: torch.Tensor) -> torch.Tensor: + return self.logits_processor(self.lm_head, hidden_states, skip_gather=True) + def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: return self.model.get_expert_mapping() @@ -1272,6 +1275,9 @@ class MiniMaxM3SparseForConditionalGeneration( def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None: return self.language_model.compute_logits(hidden_states) + def compute_logits_local(self, hidden_states: torch.Tensor) -> torch.Tensor: + return self.language_model.compute_logits_local(hidden_states) + def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: return self.language_model.get_expert_mapping() diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index ce3de50016..3bdccebfe2 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -127,6 +127,11 @@ from vllm.v1.worker.gpu.mm.lora import set_active_mm_loras from vllm.v1.worker.gpu.model_states import init_model_state from vllm.v1.worker.gpu.pool.pooling_runner import PoolingRunner from vllm.v1.worker.gpu.pp_utils import PPHandler +from vllm.v1.worker.gpu.sample.batch_shard import ( + BatchSharder, + all_to_all_logits, + gather_sampler_output, +) from vllm.v1.worker.gpu.sample.output import SamplerOutput from vllm.v1.worker.gpu.sample.prompt_logprob import PromptLogprobsWorker from vllm.v1.worker.gpu.sample.sampler import Sampler @@ -310,6 +315,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): # model_state exists (num_new_sampled_tokens_per_step from ModelState). self.sampler: Sampler | None = None self.rejection_sampler: RejectionSampler | None = None + self.batch_sharder: BatchSharder | None = None self.prompt_logprobs_worker: PromptLogprobsWorker | None = None self.structured_outputs_worker: StructuredOutputsWorker | None = None self.cudagraph_manager: ModelCudaGraphManager | None = None @@ -404,6 +410,21 @@ class GPUModelRunner(LoRAModelRunnerMixin): + self.model_state.num_new_sampled_tokens_per_step ) + if self.parallel_config.enable_batch_sharded_sampling: + if hasattr(self.model, "compute_logits_local"): + self.batch_sharder = BatchSharder( + max_num_reqs=self.max_num_reqs, + max_num_logits_per_req=self.decode_query_len, + device=self.device, + ) + logger.info("Batch-sharded sampling enabled.") + else: + logger.warning_once( + "Disabling batch-sharded sampling: %s does not implement " + "compute_logits_local", + type(self.model).__name__, + ) + # Initialize samplers. Model states may override via custom_sampler(). if self.is_last_pp_rank and not self.is_pooling_model: self.sampler = Sampler( @@ -935,7 +956,10 @@ class GPUModelRunner(LoRAModelRunnerMixin): preempted_req_ids = scheduler_output.preempted_req_ids if preempted_req_ids: finished_req_ids = finished_req_ids.union(preempted_req_ids) - for req_id in finished_req_ids: + # Sorted so every TP rank frees request slots in the same order. + # Features like batch-sharded sampling derive rank request ownership + # from the slot index. + for req_id in sorted(finished_req_ids): self._remove_request(req_id) def free_states(self, scheduler_output: SchedulerOutput) -> None: @@ -1335,8 +1359,25 @@ class GPUModelRunner(LoRAModelRunnerMixin): input_batch: InputBatch, grammar_output: GrammarOutput | None, ) -> tuple[SamplerOutput, torch.Tensor, torch.Tensor]: - sample_hidden_states = hidden_states[input_batch.logits_indices] - logits = self.model.compute_logits(sample_hidden_states) + shard_metadata = None + global_input_batch = input_batch + if self.batch_sharder is not None: + # Shard the inputs along the batch dimension to sample in parallel + # across TP ranks. + input_batch, sorted_logits_indices, grammar_output, shard_metadata = ( + self.batch_sharder.shard_sampler_inputs(input_batch, grammar_output) + ) + # The hidden states must be gathered in rank-owner-sorted order + # before computing the partial-vocab logits, so that the all-to-all + # produces full-vocab logits for just the locally-owned requests. + sample_hidden_states = hidden_states[sorted_logits_indices] + local_logits = self.model.compute_logits_local(sample_hidden_states) + logits = all_to_all_logits(local_logits, shard_metadata) + logits = logits[:, : self.vocab_size] + else: + sample_hidden_states = hidden_states[input_batch.logits_indices] + logits = self.model.compute_logits(sample_hidden_states) + if grammar_output is not None: # Apply grammar bitmask to the logits in-place. assert self.structured_outputs_worker is not None @@ -1347,7 +1388,12 @@ class GPUModelRunner(LoRAModelRunnerMixin): grammar_output.grammar_bitmask, ) - if input_batch.num_draft_tokens == 0 or self.rejection_sampler is None: + sampler_output: SamplerOutput | None + if input_batch.num_reqs == 0: + # This rank owns no requests this step. It contributes an + # all-padding block to the gather below. + sampler_output = None + elif input_batch.num_draft_tokens == 0 or self.rejection_sampler is None: assert self.sampler is not None sampler_output = self.sampler(logits, input_batch) else: @@ -1361,6 +1407,28 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.speculator.draft_logits, ) + if shard_metadata is not None: + # Gather the sharded sampler outputs from the TP ranks into a single + # sampler output. + assert self.sampler is not None + sampler_output = gather_sampler_output( + sampler_output, + shard_metadata, + device=self.device, + global_batch=global_input_batch, + local_batch=input_batch, + gather_num_nans=self.sampler.compute_nans, + logprobs_dims=self.sampler.get_logprobs_dims( + global_input_batch.idx_mapping_np, + # Rejection sampler does not return logprob token ids. + include_token_ids=( + global_input_batch.num_draft_tokens == 0 + or self.rejection_sampler is None + ), + ), + ) + + assert sampler_output is not None return sampler_output, sampler_output.num_sampled, sampler_output.num_rejected def postprocess_sampled( diff --git a/vllm/v1/worker/gpu/sample/batch_shard.py b/vllm/v1/worker/gpu/sample/batch_shard.py new file mode 100644 index 0000000000..0f484f1a17 --- /dev/null +++ b/vllm/v1/worker/gpu/sample/batch_shard.py @@ -0,0 +1,674 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from dataclasses import dataclass, replace + +import numpy as np +import torch +import torch.distributed + +from vllm.distributed.communication_op import tensor_model_parallel_all_gather +from vllm.distributed.parallel_state import get_tp_group +from vllm.triton_utils import tl, triton +from vllm.v1.core.sched.output import GrammarOutput +from vllm.v1.outputs import LogprobsTensors +from vllm.v1.worker.gpu.input_batch import InputBatch +from vllm.v1.worker.gpu.sample.output import SamplerOutput + + +@dataclass +class BatchShardMetadata: + """Collective layout for one sampling step. + + Requests (and their logits) are owner-sorted: all requests owned by + rank 0 first, then rank 1, etc. Within an owner, batch order is + preserved (stable sort). Every field is a pure function of the + replicated idx_mapping and cu_num_logits, so all ranks build identical + plans without communication — which relies on request slots + (idx_mapping) being assigned identically on every rank (see + finish_requests in the model runner). + """ + + tp_size: int + # Per-owner-rank logits counts (all-to-all send splits). + num_logits_per_rank: list[int] + num_local_logits: int + num_local_reqs: int + # The number of request entries contributed by each rank to the + # gathered result (shorter shards are padded up to this value). + max_num_reqs_per_rank: int + # [num_reqs] For batch request i, its source index in the rank-major + # gathered results. Indexing the gathered tensors with this restores + # the original batch order. + gathered_src_indices: torch.Tensor + # The number of entries occupied by each request in the gathered + # tensor. Derived from the global cu_num_logits, so each rank agrees + # ont he gather shapes, including empty shards. + max_num_logits_per_req: int + + +@triton.jit( + do_not_specialize=["num_reqs", "local_logits_start", "max_num_reqs_per_rank"] +) +def _build_shard_plan_kernel( + idx_mapping_ptr, + cu_num_logits_ptr, + query_start_loc_ptr, + seq_lens_ptr, + sorted_logits_indices_ptr, + gathered_src_indices_ptr, + local_idx_mapping_ptr, + local_cu_num_logits_ptr, + local_logits_indices_ptr, + local_expanded_idx_mapping_ptr, + local_expanded_local_pos_ptr, + local_seq_lens_ptr, + num_reqs, + local_logits_start, + max_num_reqs_per_rank, + TP_SIZE: tl.constexpr, + TP_RANK: tl.constexpr, + PADDED_NUM_REQS: tl.constexpr, + PADDED_NUM_LOGITS_PER_REQ: tl.constexpr, +): + req_idx = tl.program_id(0) + req_state_idx = tl.load(idx_mapping_ptr + req_idx) + owner = req_state_idx % TP_SIZE + num_logits = tl.load(cu_num_logits_ptr + req_idx + 1) - tl.load( + cu_num_logits_ptr + req_idx + ) + + req_block = tl.arange(0, PADDED_NUM_REQS) + req_mask = req_block < num_reqs + owners = tl.load(idx_mapping_ptr + req_block, mask=req_mask, other=0) % TP_SIZE + num_logits_per_req = tl.load( + cu_num_logits_ptr + req_block + 1, mask=req_mask, other=0 + ) - tl.load(cu_num_logits_ptr + req_block, mask=req_mask, other=0) + + # Owner-sorted layout is rank 0's requests in batch order, then rank 1's, + # etc, so a request's starting offset can be derived from counting. + lower = (owners < owner) & req_mask + earlier = (owners == owner) & (req_block < req_idx) & req_mask + + # Compute the logits row offset for the current request in the sorted + # batch. + req_logits_start = tl.sum(tl.where(lower, num_logits_per_req, 0)) + tl.sum( + tl.where(earlier, num_logits_per_req, 0) + ) + + # Copy the current request logit indices into the sorted batch. + query_start = tl.load(query_start_loc_ptr + req_idx + 1) - num_logits + logit_block = tl.arange(0, PADDED_NUM_LOGITS_PER_REQ) + logit_mask = logit_block < num_logits + logits_indices = (query_start + logit_block).to(tl.int64) + tl.store( + sorted_logits_indices_ptr + req_logits_start + logit_block, + logits_indices, + mask=logit_mask, + ) + + # Write the mapping of output tensor request indices => gathered tensor + # request indices. + # Batch request i's result lives in the padded, gathered results at + # rank_of(i) * max_num_reqs_per_rank + offset_within_rank(i). + local_req_idx = tl.sum(earlier.to(tl.int32)) + tl.store( + gathered_src_indices_ptr + req_idx, + (owner * max_num_reqs_per_rank + local_req_idx).to(tl.int64), + ) + + if req_idx == 0: + tl.store(local_cu_num_logits_ptr, 0) + + if owner == TP_RANK: + local_start = req_logits_start - local_logits_start + tl.store(local_idx_mapping_ptr + local_req_idx, req_state_idx) + tl.store(local_cu_num_logits_ptr + local_req_idx + 1, local_start + num_logits) + tl.store( + local_logits_indices_ptr + local_start + logit_block, + logits_indices, + mask=logit_mask, + ) + tl.store( + local_expanded_idx_mapping_ptr + local_start + logit_block, + req_state_idx, + mask=logit_mask, + ) + tl.store( + local_expanded_local_pos_ptr + local_start + logit_block, + logit_block, + mask=logit_mask, + ) + tl.store(local_seq_lens_ptr + local_req_idx, tl.load(seq_lens_ptr + req_idx)) + + +class BatchSharder: + """Shards the sampler inputs across TP ranks along the batch dimension.""" + + def __init__( + self, + max_num_reqs: int, + max_num_logits_per_req: int, + device: torch.device, + ): + tp_group = get_tp_group() + self.tp_rank = tp_group.rank_in_group + self.tp_size = tp_group.world_size + self.device = device + self._padded_num_reqs = triton.next_power_of_2(max_num_reqs) + self._padded_num_logits_per_req = triton.next_power_of_2(max_num_logits_per_req) + self._num_warps = max(1, min(8, self._padded_num_reqs // 128)) + + def shard_sampler_inputs( + self, + input_batch: InputBatch, + grammar_output: GrammarOutput | None, + ) -> tuple[InputBatch, torch.Tensor, GrammarOutput | None, BatchShardMetadata]: + """Owner-sort the batch and build this rank's local sampler inputs. + + Returns the local sub-batch, the owner-sorted logits_indices (gather + the sampling hidden states with these so `compute_logits_local` emits + logits in all-to-all send order), the local grammar output (None if + this rank owns none of the structured-output requests), and the shard + metadata (used for all-gathering the sampler outputs). + """ + tp_rank = self.tp_rank + tp_size = self.tp_size + num_reqs = input_batch.idx_mapping_np.shape[0] + num_logits = int(input_batch.cu_num_logits_np[-1]) + + # Deterministically assign requests to ranks, round-robin over slot + # indices so ownership stays balanced when the slot allocator fills + # low slots first (partial occupancy). + # NOTE: This mirrors the assignment used in _build_shard_plan_kernel. + req_owner_np = input_batch.idx_mapping_np % tp_size + local_req_indices_np = np.flatnonzero(req_owner_np == tp_rank) + local_idx_mapping_np = input_batch.idx_mapping_np[local_req_indices_np] + num_local_reqs = local_req_indices_np.shape[0] + num_reqs_per_rank_np = np.bincount(req_owner_np, minlength=tp_size) + max_num_reqs_per_rank = int(num_reqs_per_rank_np.max()) if num_reqs else 1 + # Derive the number of logits owned by each rank, as well as the + # local rank. + num_logits_per_req_np = np.diff(input_batch.cu_num_logits_np) + num_logits_per_rank_np = np.bincount( + req_owner_np, weights=num_logits_per_req_np, minlength=tp_size + ).astype(np.int64) + num_local_logits = int(num_logits_per_rank_np[tp_rank]) + local_logits_start = int(num_logits_per_rank_np[:tp_rank].sum()) + local_cu_num_logits_np = np.zeros(num_local_reqs + 1, dtype=np.int32) + np.cumsum( + num_logits_per_req_np[local_req_indices_np], out=local_cu_num_logits_np[1:] + ) + max_num_logits_per_req = int(num_logits_per_req_np.max()) if num_reqs else 1 + + # Shard the input batch GPU tensors. + sorted_logits_indices = torch.empty( + num_logits, dtype=torch.int64, device=self.device + ) + gathered_src_indices = torch.empty( + num_reqs, dtype=torch.int64, device=self.device + ) + local_logits_indices = torch.empty( + num_local_logits, dtype=torch.int64, device=self.device + ) + local_idx_mapping = torch.empty( + num_local_reqs, dtype=torch.int32, device=self.device + ) + local_cu_num_logits = torch.empty( + num_local_reqs + 1, dtype=torch.int32, device=self.device + ) + local_expanded_idx_mapping = torch.empty( + num_local_logits, dtype=torch.int32, device=self.device + ) + local_expanded_local_pos = torch.empty( + num_local_logits, dtype=torch.int32, device=self.device + ) + local_seq_lens = torch.empty( + num_local_reqs, dtype=torch.int32, device=self.device + ) + if num_reqs > 0: + _build_shard_plan_kernel[(num_reqs,)]( + input_batch.idx_mapping, + input_batch.cu_num_logits, + input_batch.query_start_loc, + input_batch.seq_lens, + sorted_logits_indices, + gathered_src_indices, + local_idx_mapping, + local_cu_num_logits, + local_logits_indices, + local_expanded_idx_mapping, + local_expanded_local_pos, + local_seq_lens, + num_reqs, + local_logits_start, + max_num_reqs_per_rank, + TP_SIZE=tp_size, + TP_RANK=tp_rank, + PADDED_NUM_REQS=self._padded_num_reqs, + PADDED_NUM_LOGITS_PER_REQ=self._padded_num_logits_per_req, + num_warps=self._num_warps, + ) + + # Compute the local number of draft tokens. + num_draft_tokens_per_req = None + num_draft_tokens = 0 + if input_batch.num_draft_tokens_per_req is not None: + num_draft_tokens_per_req = input_batch.num_draft_tokens_per_req[ + local_req_indices_np + ] + num_draft_tokens = int(num_draft_tokens_per_req.sum()) + + local_req_ids = [input_batch.req_ids[i] for i in local_req_indices_np.tolist()] + local_batch = replace( + input_batch, + req_ids=local_req_ids, + num_reqs=num_local_reqs, + idx_mapping=local_idx_mapping, + idx_mapping_np=local_idx_mapping_np, + expanded_idx_mapping=local_expanded_idx_mapping, + expanded_local_pos=local_expanded_local_pos, + seq_lens=local_seq_lens, + logits_indices=local_logits_indices, + cu_num_logits=local_cu_num_logits, + cu_num_logits_np=local_cu_num_logits_np, + num_draft_tokens=num_draft_tokens, + num_draft_tokens_per_req=num_draft_tokens_per_req, + ) + local_grammar_output = None + if grammar_output is not None: + local_grammar_output = _shard_grammar_output( + grammar_output, input_batch, local_req_ids + ) + metadata = BatchShardMetadata( + tp_size=tp_size, + num_logits_per_rank=num_logits_per_rank_np.tolist(), + num_local_logits=num_local_logits, + num_local_reqs=num_local_reqs, + max_num_reqs_per_rank=max_num_reqs_per_rank, + gathered_src_indices=gathered_src_indices, + max_num_logits_per_req=max_num_logits_per_req, + ) + return local_batch, sorted_logits_indices, local_grammar_output, metadata + + +def all_to_all_logits( + logits_shard: torch.Tensor, + metadata: BatchShardMetadata, +) -> torch.Tensor: + recv = logits_shard.new_empty( + metadata.tp_size * metadata.num_local_logits, logits_shard.shape[-1] + ) + torch.distributed.all_to_all_single( + recv, + logits_shard.contiguous(), + output_split_sizes=[metadata.num_local_logits] * metadata.tp_size, + input_split_sizes=metadata.num_logits_per_rank, + group=get_tp_group().device_group, + ) + shard_width = recv.shape[-1] + return ( + recv.view(metadata.tp_size, metadata.num_local_logits, shard_width) + .permute(1, 0, 2) + .reshape(metadata.num_local_logits, metadata.tp_size * shard_width) + ) + + +def _shard_grammar_output( + grammar_output: GrammarOutput, + input_batch: InputBatch, + local_req_ids: list[str], +) -> GrammarOutput | None: + owned = set(local_req_ids) + req_id_to_idx = {req_id: i for i, req_id in enumerate(input_batch.req_ids)} + cu_num_logits_np = input_batch.cu_num_logits_np + + local_ids: list[str] = [] + keep_indices: list[int] = [] + cursor = 0 + for req_id in grammar_output.structured_output_request_ids: + req_idx = req_id_to_idx[req_id] + num_req_logits = int(cu_num_logits_np[req_idx + 1] - cu_num_logits_np[req_idx]) + if req_id in owned: + local_ids.append(req_id) + keep_indices.extend(range(cursor, cursor + num_req_logits)) + cursor += num_req_logits + if not local_ids: + return None + return GrammarOutput( + structured_output_request_ids=local_ids, + grammar_bitmask=grammar_output.grammar_bitmask[keep_indices], + ) + + +@triton.jit( + do_not_specialize=["max_num_logits_per_req", "num_src_cols", "packed_stride"] +) +def _pack_sampler_output_kernel( + packed_ptr, + packed_stride, + sampled_token_ids_ptr, + sampled_token_ids_stride, + num_sampled_ptr, + num_rejected_ptr, + num_nans_ptr, + local_cu_num_logits_ptr, + max_num_logits_per_req, + num_src_cols, + BLOCK_SIZE: tl.constexpr, +): + req_idx = tl.program_id(0) + cols = tl.arange(0, BLOCK_SIZE) + row_ptr = packed_ptr + req_idx * packed_stride + + token_ids = tl.load( + sampled_token_ids_ptr + req_idx * sampled_token_ids_stride + cols, + mask=cols < num_src_cols, + other=0, + ) + tl.store(row_ptr + cols, token_ids, mask=cols < max_num_logits_per_req) + num_sampled = tl.load(num_sampled_ptr + req_idx) + tl.store(row_ptr + max_num_logits_per_req, num_sampled.to(tl.int64)) + num_rejected = tl.load(num_rejected_ptr + req_idx) + tl.store(row_ptr + max_num_logits_per_req + 1, num_rejected.to(tl.int64)) + if num_nans_ptr is not None: + # num_nans is per logits row; reduce it per request. The replicated + # samplers report per-row counts that downstream zips per request, + # so a per-request sum is the well-defined equivalent. + start = tl.load(local_cu_num_logits_ptr + req_idx) + num_req_logits = tl.load(local_cu_num_logits_ptr + req_idx + 1) - start + nans = tl.load(num_nans_ptr + start + cols, mask=cols < num_req_logits, other=0) + tl.store( + row_ptr + max_num_logits_per_req + 2, tl.sum(nans.to(tl.int64), axis=0) + ) + + +@triton.jit(do_not_specialize=["max_num_logits_per_req", "gathered_stride"]) +def _unpack_gathered_output_kernel( + gathered_ptr, + gathered_stride, + gathered_src_indices_ptr, + sampled_token_ids_ptr, + num_sampled_ptr, + num_rejected_ptr, + num_nans_ptr, + max_num_logits_per_req, + BLOCK_SIZE: tl.constexpr, +): + req_idx = tl.program_id(0) + src = tl.load(gathered_src_indices_ptr + req_idx) + row_ptr = gathered_ptr + src * gathered_stride + cols = tl.arange(0, BLOCK_SIZE) + mask = cols < max_num_logits_per_req + + token_ids = tl.load(row_ptr + cols, mask=mask) + tl.store( + sampled_token_ids_ptr + req_idx * max_num_logits_per_req + cols, + token_ids, + mask=mask, + ) + num_sampled = tl.load(row_ptr + max_num_logits_per_req) + tl.store(num_sampled_ptr + req_idx, num_sampled.to(tl.int32)) + num_rejected = tl.load(row_ptr + max_num_logits_per_req + 1) + tl.store(num_rejected_ptr + req_idx, num_rejected.to(tl.int32)) + if num_nans_ptr is not None: + num_nans = tl.load(row_ptr + max_num_logits_per_req + 2) + tl.store(num_nans_ptr + req_idx, num_nans.to(tl.int32)) + + +@triton.jit( + do_not_specialize=[ + "max_num_logits_per_req", + "num_logprob_cols", + "num_src_cols", + "send_ids_stride", + "send_logprobs_stride", + "src_ids_stride", + "src_logprobs_stride", + ] +) +def _pack_logprobs_kernel( + send_ids_ptr, + send_ids_stride, + send_logprobs_ptr, + send_logprobs_stride, + src_ids_ptr, + src_ids_stride, + src_logprobs_ptr, + src_logprobs_stride, + selected_token_ranks_ptr, + local_cu_num_logits_ptr, + max_num_logits_per_req, + num_logprob_cols, + num_src_cols, + BLOCK_SIZE: tl.constexpr, +): + req_idx = tl.program_id(0) + pos = tl.program_id(1) + start = tl.load(local_cu_num_logits_ptr + req_idx) + if pos < tl.load(local_cu_num_logits_ptr + req_idx + 1) - start: + src = start + pos + dst = req_idx * max_num_logits_per_req + pos + cols = tl.arange(0, BLOCK_SIZE) + # A rank whose sub-batch has no draft tokens runs the regular sampler + # while others run the rejection sampler, and their column counts can + # differ (e.g. logprob_token_ids widening); the two masks pad or + # truncate to the agreed width. + src_mask = cols < num_src_cols + dst_mask = cols < num_logprob_cols + ids = tl.load(src_ids_ptr + src * src_ids_stride + cols, mask=src_mask, other=0) + logprobs = tl.load( + src_logprobs_ptr + src * src_logprobs_stride + cols, + mask=src_mask, + other=float("-inf"), + ) + ids_row = send_ids_ptr + dst * send_ids_stride + tl.store(ids_row + cols, ids.to(tl.int64), mask=dst_mask) + rank = tl.load(selected_token_ranks_ptr + src) + tl.store(ids_row + num_logprob_cols, rank.to(tl.int64)) + tl.store( + send_logprobs_ptr + dst * send_logprobs_stride + cols, + logprobs.to(tl.float32), + mask=dst_mask, + ) + + +@triton.jit( + do_not_specialize=[ + "max_num_logits_per_req", + "num_logprob_cols", + "gathered_ids_stride", + "gathered_logprobs_stride", + ] +) +def _unpack_logprobs_kernel( + gathered_ids_ptr, + gathered_ids_stride, + gathered_logprobs_ptr, + gathered_logprobs_stride, + gathered_src_indices_ptr, + cu_num_logits_ptr, + logprob_token_ids_ptr, + logprobs_ptr, + selected_token_ranks_ptr, + max_num_logits_per_req, + num_logprob_cols, + BLOCK_SIZE: tl.constexpr, +): + req_idx = tl.program_id(0) + pos = tl.program_id(1) + start = tl.load(cu_num_logits_ptr + req_idx) + if pos < tl.load(cu_num_logits_ptr + req_idx + 1) - start: + src = tl.load(gathered_src_indices_ptr + req_idx) * max_num_logits_per_req + pos + dst = start + pos + cols = tl.arange(0, BLOCK_SIZE) + mask = cols < num_logprob_cols + src_ids_row = gathered_ids_ptr + src * gathered_ids_stride + ids = tl.load(src_ids_row + cols, mask=mask) + tl.store(logprob_token_ids_ptr + dst * num_logprob_cols + cols, ids, mask=mask) + rank = tl.load(src_ids_row + num_logprob_cols) + tl.store(selected_token_ranks_ptr + dst, rank) + logprobs = tl.load( + gathered_logprobs_ptr + src * gathered_logprobs_stride + cols, mask=mask + ) + tl.store(logprobs_ptr + dst * num_logprob_cols + cols, logprobs, mask=mask) + + +def _gather_logprobs_tensors( + local_output: SamplerOutput | None, + metadata: BatchShardMetadata, + global_batch: InputBatch, + local_batch: InputBatch, + logprobs_dims: tuple[int, int], + device: torch.device, +) -> LogprobsTensors: + num_logprobs, max_token_ids = logprobs_dims + num_logprob_cols = 1 + max(num_logprobs, max_token_ids) + max_num_logits_per_req = metadata.max_num_logits_per_req + num_send_rows = metadata.max_num_reqs_per_rank * max_num_logits_per_req + block_size = triton.next_power_of_2(num_logprob_cols) + + # Padded slots are never read back: gathered_src_indices only addresses + # rows that an owner rank wrote. + send_ids_ranks = torch.empty( + num_send_rows, num_logprob_cols + 1, dtype=torch.int64, device=device + ) + send_logprobs = torch.empty( + num_send_rows, num_logprob_cols, dtype=torch.float32, device=device + ) + if local_output is not None and local_output.logprobs_tensors is not None: + lp = local_output.logprobs_tensors + _pack_logprobs_kernel[(metadata.num_local_reqs, max_num_logits_per_req)]( + send_ids_ranks, + send_ids_ranks.stride(0), + send_logprobs, + send_logprobs.stride(0), + lp.logprob_token_ids, + lp.logprob_token_ids.stride(0), + lp.logprobs, + lp.logprobs.stride(0), + lp.selected_token_ranks, + local_batch.cu_num_logits, + max_num_logits_per_req, + num_logprob_cols, + lp.logprob_token_ids.shape[1], + BLOCK_SIZE=block_size, + ) + + gathered_ids_ranks = tensor_model_parallel_all_gather(send_ids_ranks, dim=0) + gathered_logprobs = tensor_model_parallel_all_gather(send_logprobs, dim=0) + + num_reqs = global_batch.num_reqs + num_logits = int(global_batch.cu_num_logits_np[-1]) + logprob_token_ids = torch.empty( + num_logits, num_logprob_cols, dtype=torch.int64, device=device + ) + logprobs = torch.empty( + num_logits, num_logprob_cols, dtype=torch.float32, device=device + ) + selected_token_ranks = torch.empty(num_logits, dtype=torch.int64, device=device) + _unpack_logprobs_kernel[(num_reqs, max_num_logits_per_req)]( + gathered_ids_ranks, + gathered_ids_ranks.stride(0), + gathered_logprobs, + gathered_logprobs.stride(0), + metadata.gathered_src_indices, + global_batch.cu_num_logits, + logprob_token_ids, + logprobs, + selected_token_ranks, + max_num_logits_per_req, + num_logprob_cols, + BLOCK_SIZE=block_size, + ) + return LogprobsTensors( + logprob_token_ids=logprob_token_ids, + logprobs=logprobs, + selected_token_ranks=selected_token_ranks, + cu_num_generated_tokens=( + global_batch.cu_num_logits_np.tolist() if num_logits != num_reqs else None + ), + ) + + +def gather_sampler_output( + local_output: SamplerOutput | None, + metadata: BatchShardMetadata, + device: torch.device, + global_batch: InputBatch, + local_batch: InputBatch, + gather_num_nans: bool = False, + logprobs_dims: tuple[int, int] | None = None, +) -> SamplerOutput: + max_num_logits_per_req = metadata.max_num_logits_per_req + num_packed_cols = max_num_logits_per_req + 2 + (1 if gather_num_nans else 0) + block_size = triton.next_power_of_2(max_num_logits_per_req) + + # Pack the sampler output tensors (excluding logprobs) into a single + # tensor for the all-gather. + packed = torch.empty( + metadata.max_num_reqs_per_rank, + num_packed_cols, + dtype=torch.int64, + device=device, + ) + if local_output is not None: + assert local_output.num_sampled is not None + assert local_output.num_rejected is not None + assert not gather_num_nans or local_output.num_nans is not None + num_src_cols = min( + local_output.sampled_token_ids.shape[1], max_num_logits_per_req + ) + _pack_sampler_output_kernel[(metadata.num_local_reqs,)]( + packed, + packed.stride(0), + local_output.sampled_token_ids, + local_output.sampled_token_ids.stride(0), + local_output.num_sampled, + local_output.num_rejected, + local_output.num_nans if gather_num_nans else None, + local_batch.cu_num_logits if gather_num_nans else None, + max_num_logits_per_req, + num_src_cols, + BLOCK_SIZE=block_size, + ) + + # All-gather the sampler output tensors (excluding logprobs). + gathered = tensor_model_parallel_all_gather(packed, dim=0) + # All-gather the logprobs tensors. + logprobs_tensors = None + if logprobs_dims is not None: + logprobs_tensors = _gather_logprobs_tensors( + local_output, metadata, global_batch, local_batch, logprobs_dims, device + ) + + # Unpack the gathered tensor into a sampler output object. + num_reqs = metadata.gathered_src_indices.shape[0] + sampled_token_ids = torch.empty( + num_reqs, max_num_logits_per_req, dtype=torch.int64, device=device + ) + num_sampled = torch.empty(num_reqs, dtype=torch.int32, device=device) + num_rejected = torch.empty(num_reqs, dtype=torch.int32, device=device) + num_nans = ( + torch.empty(num_reqs, dtype=torch.int32, device=device) + if gather_num_nans + else None + ) + _unpack_gathered_output_kernel[(num_reqs,)]( + gathered, + gathered.stride(0), + metadata.gathered_src_indices, + sampled_token_ids, + num_sampled, + num_rejected, + num_nans, + max_num_logits_per_req, + BLOCK_SIZE=block_size, + ) + return SamplerOutput( + sampled_token_ids=sampled_token_ids, + logprobs_tensors=logprobs_tensors, + num_nans=num_nans, + num_sampled=num_sampled, + num_rejected=num_rejected, + ) diff --git a/vllm/v1/worker/gpu/sample/sampler.py b/vllm/v1/worker/gpu/sample/sampler.py index 9e65d9a304..d3205a3dae 100644 --- a/vllm/v1/worker/gpu/sample/sampler.py +++ b/vllm/v1/worker/gpu/sample/sampler.py @@ -103,6 +103,22 @@ class Sampler: if self.trace_replay_state is not None: self.trace_replay_state.apply_staged_writes() + def get_logprobs_dims( + self, idx_mapping_np: np.ndarray, include_token_ids: bool = True + ) -> tuple[int, int] | None: + """(num_logprobs, max_per_req_token_ids) for the given requests, or + None when none of them want logprobs.""" + max_num_logprobs = self.sampling_states.max_num_logprobs(idx_mapping_np) + max_token_ids = ( + self.logprob_token_ids_state.max_num_token_ids(idx_mapping_np) + if include_token_ids + else 0 + ) + if max_num_logprobs == NO_LOGPROBS and max_token_ids == 0: + return None + num_logprobs = max_num_logprobs if max_num_logprobs != NO_LOGPROBS else 0 + return num_logprobs, max_token_ids + def __call__( self, logits: torch.Tensor, @@ -120,11 +136,7 @@ class Sampler: # that num_nans is computed before applying penalties and temperature. num_nans = get_num_nans(logits) if self.compute_nans else None - max_num_logprobs = self.sampling_states.max_num_logprobs(idx_mapping_np) - max_per_req_token_ids = self.logprob_token_ids_state.max_num_token_ids( - idx_mapping_np - ) - return_logprobs = max_num_logprobs != NO_LOGPROBS or max_per_req_token_ids > 0 + logprobs_dims = self.get_logprobs_dims(idx_mapping_np) sampled, processed_logits = self.sample( logits, @@ -134,7 +146,7 @@ class Sampler: pos, input_ids, expanded_local_pos, - return_logprobs=return_logprobs, + return_logprobs=logprobs_dims is not None, ) if self.trace_replay_state is not None: @@ -142,12 +154,12 @@ class Sampler: # computed logprobs reflect the real distribution of the forced token. self.trace_replay_state.apply_trace(sampled, idx_mapping) - if return_logprobs: + if logprobs_dims is not None: + num_logprobs, max_per_req_token_ids = logprobs_dims if self.logprobs_mode in PROCESSED_LOGPROBS_MODES: logits = processed_logits expanded_logits = logits.shape[0] != idx_mapping_np.shape[0] cu_num_logits = cu_num_logits_np.tolist() if expanded_logits else None - num_logprobs = max_num_logprobs if max_num_logprobs != NO_LOGPROBS else 0 logprobs_tensors = compute_topk_scores( logits, num_logprobs,