[Model Runner V2] batch-sharded sample (#50465)

Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
Giancarlo Delfin
2026-08-24 15:49:57 -07:00
committed by GitHub
parent 0d7d5ed0b2
commit d154d90d6c
19 changed files with 1781 additions and 22 deletions
+6 -1
View File
@@ -63,7 +63,7 @@ steps:
- label: ":nvidia: (L4) Model Runner V2 Distributed"
key: model-runner-v2-distributed-2-gpus
timeout_in_minutes: 30
timeout_in_minutes: 40
working_dir: "/vllm-workspace/tests"
num_devices: 2
source_file_dependencies:
@@ -72,6 +72,8 @@ steps:
- tests/basic_correctness/test_basic_correctness.py
- tests/v1/distributed/test_async_llm_dp.py
- tests/v1/distributed/test_eagle_dp.py
- tests/v1/e2e/general/test_sharded_sampling.py
- tests/v1/e2e/spec_decode/test_sharded_sampling.py
commands:
- set -x
- export VLLM_USE_V2_MODEL_RUNNER=1
@@ -81,6 +83,9 @@ steps:
- export NCCL_CUMEM_HOST_ENABLE=0
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py -k "not ray"
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
- pytest -v -s v1/e2e/general/test_sharded_sampling.py
- pytest -v -s v1/e2e/spec_decode/test_sharded_sampling.py
mirror:
amd:
label: ":amd: (MI300) Model Runner V2 Distributed"
@@ -0,0 +1,162 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Batch-sharded sampling must be a drop-in for replicated sampling.
The comparison is tolerance-based, not bitwise: engine boots land in
slightly different kernel/collective states (measured: same-state boots are
bitwise identical across the two modes; cross-state boots differ by up to
~0.25 logprob and can flip a near-tie token). A real sharding bug — rows
routed to the wrong request, misaligned gathers — produces many-nats
divergence and scrambled top-k sets, far outside these bounds."""
import pytest
from tests.utils import multi_gpu_test
from vllm import LLM, SamplingParams
from vllm.distributed import cleanup_dist_env_and_memory
MODEL = "meta-llama/Llama-3.2-1B-Instruct"
# Boot-state numeric noise bound (measured <= 0.25 on identical contexts).
LOGPROB_TOL = 0.5
# Near-tie greedy/seeded flips from boot-state noise; measured <= 1 per
# engine boot across ~16 completions, plus one of headroom.
MAX_DIVERGENT_PROMPTS = 3
PROMPTS = [
"The capital of France is",
"The three primary colors are",
"Write a haiku about the ocean.",
"1, 1, 2, 3, 5, 8,",
"The chemical symbol for gold is",
"Once upon a time, in a distant kingdom,",
"Python's GIL stands for",
"The fastest land animal is",
]
# Exercise greedy, seeded random, top-k/top-p, penalties, and logprobs.
SAMPLING_PARAMS = [
SamplingParams(temperature=0.0, max_tokens=32, logprobs=5, prompt_logprobs=1),
SamplingParams(temperature=0.8, seed=42, top_p=0.9, max_tokens=32, logprobs=3),
SamplingParams(temperature=0.7, seed=7, top_k=50, min_p=0.05, max_tokens=32),
SamplingParams(
temperature=0.9,
seed=123,
frequency_penalty=0.5,
repetition_penalty=1.2,
max_tokens=32,
logprobs=2,
),
]
def _is_sharded_sampling_active(worker) -> bool:
return worker.model_runner.batch_sharder is not None
def _generate(monkeypatch: pytest.MonkeyPatch, disable_sharding: bool):
llm = LLM(
model=MODEL,
tensor_parallel_size=2,
enable_batch_sharded_sampling=not disable_sharding,
max_model_len=1024,
# More requests than ranks, so round-robin slot ownership splits the
# batch across both ranks.
max_num_seqs=len(PROMPTS),
enforce_eager=True,
enable_prefix_caching=False,
gpu_memory_utilization=0.7,
# Timing-based kernel selection is the dominant source of cross-boot
# numeric noise; disable it so the comparison bounds stay tight.
kernel_config={"enable_flashinfer_autotune": False},
)
try:
# Guard against a vacuous comparison: verify every worker is in the
# intended sampling mode.
modes = llm.llm_engine.collective_rpc(_is_sharded_sampling_active)
assert all(mode == (not disable_sharding) for mode in modes), modes
params = [
SAMPLING_PARAMS[i % len(SAMPLING_PARAMS)] for i in range(len(PROMPTS))
]
# Two waves: the second reuses the request slots freed by the first
# (in whatever order requests finished), covering the cross-rank
# slot-recycling path that request ownership derives from.
wave1 = llm.generate(PROMPTS, params)
wave2 = llm.generate(PROMPTS[::-1], params[::-1])
return wave1 + wave2
finally:
del llm
cleanup_dist_env_and_memory()
def _assert_logprob_dicts_close(ref_lps, out_lps, what: str) -> None:
# Allow one boundary entry of the top-k set to swap at a near-tie.
common = set(ref_lps) & set(out_lps)
assert len(common) >= max(len(ref_lps), len(out_lps)) - 1, (
f"{what}: top-k token sets diverge: {sorted(ref_lps)} vs {sorted(out_lps)}"
)
for token_id in common:
diff = abs(ref_lps[token_id].logprob - out_lps[token_id].logprob)
assert diff <= LOGPROB_TOL, (
f"{what}[{token_id}]: logprob diff {diff} "
f"({ref_lps[token_id].logprob} vs {out_lps[token_id].logprob})"
)
@multi_gpu_test(num_gpus=2)
def test_sharded_sampling_outputs_match(monkeypatch: pytest.MonkeyPatch):
"""Generation, logprobs, and prompt logprobs match between batch-sharded
sampling and the replicated fallback, up to measured boot-state noise."""
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
# Required for the collective_rpc mode check below.
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
ref_outputs = _generate(monkeypatch, disable_sharding=True)
shard_outputs = _generate(monkeypatch, disable_sharding=False)
assert len(ref_outputs) == len(shard_outputs) == 2 * len(PROMPTS)
num_divergent = 0
for i, (ref, out) in enumerate(zip(ref_outputs, shard_outputs)):
ref_completion = ref.outputs[0]
out_completion = out.outputs[0]
ref_ids = list(ref_completion.token_ids)
out_ids = list(out_completion.token_ids)
# Logits are only comparable at positions whose context (prompt +
# previously sampled tokens) is identical: every position up to and
# including the first divergence.
prefix = 0
while prefix < min(len(ref_ids), len(out_ids)):
if ref_ids[prefix] != out_ids[prefix]:
break
prefix += 1
if ref_ids != out_ids:
num_divergent += 1
if ref_completion.logprobs is not None:
assert out_completion.logprobs is not None
comparable = min(prefix + 1, len(ref_ids), len(out_ids))
for pos in range(comparable):
_assert_logprob_dicts_close(
ref_completion.logprobs[pos],
out_completion.logprobs[pos],
f"prompt {i} logprobs[{pos}]",
)
# The prompt is fixed, so every prompt-logprob position is comparable.
if ref.prompt_logprobs is not None:
assert out.prompt_logprobs is not None
for pos, (ref_lps, out_lps) in enumerate(
zip(ref.prompt_logprobs, out.prompt_logprobs)
):
if ref_lps is None or out_lps is None:
assert ref_lps is None and out_lps is None
continue
_assert_logprob_dicts_close(
ref_lps, out_lps, f"prompt {i} prompt_logprobs[{pos}]"
)
assert num_divergent <= MAX_DIVERGENT_PROMPTS, (
f"{num_divergent}/{2 * len(PROMPTS)} prompts diverged: beyond near-tie "
"boot-state noise; sharded sampling is likely misrouting requests"
)
@@ -0,0 +1,102 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Batch-sharded sampling must be a drop-in for replicated sampling under
speculative decoding.
The non-spec arm of this claim lives in tests/v1/e2e/general/
test_sharded_sampling.py. Spec decode is the harder case: draft tokens give
each request a different number of logits rows, so the shard plan's
per-request arithmetic and the rejection sampler are both exercised.
"""
import pytest
import torch
from tests.utils import wait_for_memory_to_settle
from vllm import LLM, SamplingParams
from vllm.distributed import cleanup_dist_env_and_memory
from .utils import _skip_if_insufficient_gpus_for_tp, get_test_prompts, greedy_sampling
def _is_sharded_sampling_active(worker) -> bool:
return worker.model_runner.batch_sharder is not None
def test_mtp_sharded_sampling_equivalence(monkeypatch: pytest.MonkeyPatch):
"""Batch-sharded sampling must be a bit-exact drop-in for replicated
sampling under MTP spec decoding: the collectives move the same logits
bytes, Gumbel keys derive from (request slot, position, seed), and slot
assignment is rank-deterministic. Both runs here are spec decode with
identical math, so outputs must match exactly."""
tp_size = 2
_skip_if_insufficient_gpus_for_tp(tp_size)
model_name = "Qwen/Qwen3.5-0.8B-Base"
test_prompts = get_test_prompts(mm_enabled=False, num_prompts=20)
with monkeypatch.context() as m:
# Batch-sharded sampling only exists in the V2 model runner.
m.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
# Required for the collective_rpc mode check below.
m.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
def run(disable_sharding: bool):
llm = LLM(
model=model_name,
tensor_parallel_size=tp_size,
enable_batch_sharded_sampling=not disable_sharding,
speculative_config={
"method": "mtp",
"num_speculative_tokens": 1,
"max_model_len": 2048,
},
max_model_len=2048,
limit_mm_per_prompt={"image": 0, "video": 0},
# More requests than ranks, so round-robin slot ownership
# splits the batch across both ranks.
max_num_seqs=len(test_prompts),
# Timing-based kernel selection is the dominant source of
# cross-boot numeric noise; disable for a tight comparison.
kernel_config={"enable_flashinfer_autotune": False},
)
try:
# Guard against a vacuous comparison: verify every worker is
# in the intended sampling mode.
modes = llm.llm_engine.collective_rpc(_is_sharded_sampling_active)
assert all(mode == (not disable_sharding) for mode in modes)
greedy = llm.chat(test_prompts, greedy_sampling())
seeded = llm.chat(
test_prompts,
SamplingParams(temperature=1.0, seed=33, max_tokens=10),
)
return greedy, seeded
finally:
del llm
torch.accelerator.empty_cache()
cleanup_dist_env_and_memory()
wait_for_memory_to_settle()
ref_greedy, ref_seeded = run(disable_sharding=True)
shard_greedy, shard_seeded = run(disable_sharding=False)
# Engine boots are not bitwise deterministic (kernel/collective
# selection shifts logits by ~1 ulp, flipping near-tie tokens), so
# allow a small number of divergent prompts. A sharding bug produces
# wholesale divergence, not isolated near-tie flips.
for name, ref_outputs, shard_outputs in (
("greedy", ref_greedy, shard_greedy),
("seeded", ref_seeded, shard_seeded),
):
num_divergent = 0
for i, (ref, out) in enumerate(zip(ref_outputs, shard_outputs)):
if list(ref.outputs[0].token_ids) != list(out.outputs[0].token_ids):
num_divergent += 1
print(
f"{name} prompt {i} diverged:\n"
f" replicated: {ref.outputs[0].text!r}\n"
f" sharded: {out.outputs[0].text!r}"
)
assert num_divergent <= 2, (
f"{name}: {num_divergent}/{len(ref_outputs)} prompts diverged, "
"beyond near-tie boot noise"
)
+613
View File
@@ -0,0 +1,613 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Routing-math tests for batch-sharded sampling.
These simulate every TP rank in-process (no torch.distributed): the local
sub-batch, the all-to-all send/recv layout, and the padded result gather must
compose to a bit-exact round trip of the replicated computation.
"""
from types import SimpleNamespace
from unittest import mock
import numpy as np
import pytest
import torch
from vllm.v1.core.sched.output import GrammarOutput
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.sample import batch_shard
from vllm.v1.worker.gpu.sample.batch_shard import (
BatchSharder,
_shard_grammar_output,
)
DEVICE = "cuda"
# The owner-sorted layout is expanded by a Triton kernel, so the routing math
# can only be exercised on device.
requires_cuda = pytest.mark.skipif(
not torch.cuda.is_available(), reason="requires CUDA"
)
def _np(t: torch.Tensor) -> np.ndarray:
return t.cpu().numpy()
def _make_batch(
rng: np.random.Generator,
num_reqs: int,
max_num_reqs: int,
max_spec: int,
slot_pool: np.ndarray | None = None,
) -> InputBatch:
"""Consistent batch with random unique request slots, 1 + k_i logits
rows per request at the tail of each query segment."""
if slot_pool is None:
slot_pool = np.arange(max_num_reqs)
slots = rng.choice(slot_pool, size=num_reqs, replace=False)
slots = slots.astype(np.int32)
k = rng.integers(0, max_spec + 1, size=num_reqs)
num_logits_per_req = (1 + k).astype(np.int32)
cu = np.zeros(num_reqs + 1, dtype=np.int32)
np.cumsum(num_logits_per_req, out=cu[1:])
query_lens = num_logits_per_req + rng.integers(0, 4, size=num_reqs).astype(np.int32)
qsl = np.zeros(num_reqs + 1, dtype=np.int32)
np.cumsum(query_lens, out=qsl[1:])
num_tokens = int(qsl[-1])
logits_indices_np = np.concatenate(
[
np.arange(qsl[i + 1] - num_logits_per_req[i], qsl[i + 1])
for i in range(num_reqs)
]
).astype(np.int64)
expanded_idx_np = np.repeat(slots, num_logits_per_req)
expanded_pos_np = np.concatenate(
[np.arange(n, dtype=np.int32) for n in num_logits_per_req]
)
seq_lens_np = (query_lens + rng.integers(0, 100, size=num_reqs)).astype(np.int32)
def dev(a: np.ndarray) -> torch.Tensor:
return torch.from_numpy(a).to(DEVICE)
return InputBatch(
req_ids=[f"req-{i}" for i in range(num_reqs)],
num_reqs=num_reqs,
num_reqs_after_padding=num_reqs,
idx_mapping=dev(slots),
idx_mapping_np=slots,
expanded_idx_mapping=dev(expanded_idx_np),
expanded_local_pos=dev(expanded_pos_np),
num_scheduled_tokens=query_lens,
num_tokens=num_tokens,
num_tokens_after_padding=num_tokens,
num_draft_tokens=int(k.sum()),
num_draft_tokens_per_req=k.astype(np.int32),
query_start_loc=dev(qsl),
query_start_loc_np=qsl,
seq_lens=dev(seq_lens_np),
seq_lens_cpu_upper_bound=torch.from_numpy(seq_lens_np.copy()),
dcp_local_seq_lens=None,
num_computed_tokens_np=np.zeros(num_reqs, dtype=np.int32),
prefill_len_np=np.zeros(num_reqs, dtype=np.int32),
num_computed_prefill_tokens_np=np.zeros(num_reqs, dtype=np.int32),
is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_),
has_prefill=False,
max_seq_len_np=None,
input_ids=torch.zeros(num_tokens, dtype=torch.int32, device=DEVICE),
positions=torch.arange(num_tokens, dtype=torch.int64, device=DEVICE),
is_padding=torch.zeros(num_tokens, dtype=torch.bool, device=DEVICE),
logits_indices=dev(logits_indices_np),
cu_num_logits=dev(cu),
cu_num_logits_np=cu,
has_structured_output_reqs=False,
prompt_lens=None,
)
def _shard_all_ranks(batch: InputBatch, max_num_reqs: int, tp_size: int):
"""Simulate every TP rank by faking get_tp_group() per BatchSharder."""
results = []
for rank in range(tp_size):
fake_group = SimpleNamespace(rank_in_group=rank, world_size=tp_size)
with mock.patch.object(batch_shard, "get_tp_group", return_value=fake_group):
sharder = BatchSharder(
max_num_reqs=max_num_reqs,
max_num_logits_per_req=8,
device=torch.device(DEVICE),
)
local_batch, sorted_logits_indices, _, metadata = sharder.shard_sampler_inputs(
batch, None
)
results.append((local_batch, sorted_logits_indices, metadata))
return results
def _owned_batch_indices(local_batch: InputBatch) -> np.ndarray:
return np.array([int(r.split("-")[1]) for r in local_batch.req_ids], dtype=np.int64)
def _owned_rows(cu: np.ndarray, owned: np.ndarray) -> np.ndarray:
if len(owned) == 0:
return np.array([], dtype=np.int64)
return np.concatenate([np.arange(cu[r], cu[r + 1]) for r in owned])
@requires_cuda
@pytest.mark.parametrize("tp_size", [2, 4, 8])
@pytest.mark.parametrize("seed", [0, 1])
def test_local_batch_partition(tp_size: int, seed: int):
"""Local sub-batches partition the batch by owner, preserve batch order
within an owner, and carry exactly the owned requests' entries of every
per-request and per-row array."""
rng = np.random.default_rng(seed)
max_num_reqs = 64
batch = _make_batch(rng, num_reqs=23, max_num_reqs=max_num_reqs, max_spec=3)
num_logits = int(batch.cu_num_logits_np[-1])
results = _shard_all_ranks(batch, max_num_reqs, tp_size)
cu = batch.cu_num_logits_np
metadata0 = results[0][2]
assert sum(metadata0.num_logits_per_rank) == num_logits
# Gather width = max logits rows of any request (bounds num_sampled).
assert metadata0.max_num_logits_per_req == int(np.diff(cu).max())
all_owned: list[int] = []
for rank, (local, _, metadata) in enumerate(results):
owned = _owned_batch_indices(local)
assert local.num_reqs == len(owned) == metadata.num_local_reqs
assert (batch.idx_mapping_np[owned] % tp_size == rank).all()
# Stable sort: batch order is preserved within an owner.
assert (np.diff(owned) > 0).all()
all_owned.extend(owned.tolist())
rows = _owned_rows(cu, owned)
assert (
metadata.num_local_logits
== len(rows)
== metadata0.num_logits_per_rank[rank]
)
expected_idx = batch.idx_mapping_np[owned]
assert (local.idx_mapping_np == expected_idx).all()
assert _np(local.idx_mapping).tolist() == expected_idx.tolist()
assert (np.diff(local.cu_num_logits_np) == np.diff(cu)[owned]).all()
assert _np(local.cu_num_logits).tolist() == local.cu_num_logits_np.tolist()
assert (
_np(local.expanded_idx_mapping) == _np(batch.expanded_idx_mapping)[rows]
).all()
assert (
_np(local.expanded_local_pos) == _np(batch.expanded_local_pos)[rows]
).all()
assert (_np(local.logits_indices) == _np(batch.logits_indices)[rows]).all()
assert (_np(local.seq_lens) == _np(batch.seq_lens)[owned]).all()
assert batch.num_draft_tokens_per_req is not None
assert local.num_draft_tokens_per_req is not None
assert (
local.num_draft_tokens_per_req == batch.num_draft_tokens_per_req[owned]
).all()
assert local.num_draft_tokens == int(
batch.num_draft_tokens_per_req[owned].sum()
)
# Triton specializes kernels on pointer alignment: every packed-buffer
# view fed to kernels must stay 16-byte aligned across batch shapes,
# or serving JITs new kernel variants mid-collective.
for t in (
local.idx_mapping,
local.expanded_idx_mapping,
local.expanded_local_pos,
local.logits_indices,
local.cu_num_logits,
metadata.gathered_src_indices,
):
assert t.data_ptr() % 16 == 0
assert sorted(all_owned) == list(range(batch.num_reqs))
# sorted_logits_indices is identical on all ranks: the owner-sorted logits row
# positions, i.e. logits_indices permuted rank-major.
sorted_logits_indices = _np(results[0][1])
for _, gi, _ in results:
assert (_np(gi) == sorted_logits_indices).all()
rows_all = np.concatenate(
[_owned_rows(cu, _owned_batch_indices(local)) for local, _, _ in results]
)
assert (
sorted_logits_indices.tolist() == _np(batch.logits_indices)[rows_all].tolist()
)
@requires_cuda
@pytest.mark.parametrize("tp_size", [2, 4])
def test_all_to_all_reassembly_simulated(tp_size: int):
"""Vocab shards computed from owner-sorted hidden states and routed
through the simulated all-to-all reassemble into exactly the full-vocab
logits of each rank's owned rows."""
rng = np.random.default_rng(7)
max_num_reqs = 32
batch = _make_batch(rng, num_reqs=11, max_num_reqs=max_num_reqs, max_spec=4)
vocab = 64 * tp_size
shard_w = vocab // tp_size
token_logits = torch.randn(batch.num_tokens, vocab, device=DEVICE)
results = _shard_all_ranks(batch, max_num_reqs, tp_size)
sorted_logits_indices = results[0][1]
num_logits_per_rank = results[0][2].num_logits_per_rank
bounds = np.zeros(tp_size + 1, dtype=np.int64)
np.cumsum(num_logits_per_rank, out=bounds[1:])
# send_r = rank r's vocab shard of the owner-sorted rows, with no further
# permute; chunk d of send_r goes to rank d.
sends = [
token_logits[sorted_logits_indices][:, r * shard_w : (r + 1) * shard_w]
for r in range(tp_size)
]
for rank, (local, _, metadata) in enumerate(results):
recv = torch.cat(
[sends[src][bounds[rank] : bounds[rank + 1]] for src in range(tp_size)]
)
# Reassemble source-rank-major chunks into full-vocab rows, mirroring
# all_to_all_logits (source-rank order is vocab-shard order).
reassembled = (
recv.view(tp_size, metadata.num_local_logits, shard_w)
.permute(1, 0, 2)
.reshape(metadata.num_local_logits, vocab)
)
expected = token_logits[local.logits_indices]
assert torch.equal(reassembled, expected)
@requires_cuda
def test_shard_no_spec():
"""Batches without draft tokens (one logits row per request) degenerate
cleanly: rows mirror requests."""
rng = np.random.default_rng(11)
tp_size = 4
max_num_reqs = 32
batch = _make_batch(rng, num_reqs=10, max_num_reqs=max_num_reqs, max_spec=0)
results = _shard_all_ranks(batch, max_num_reqs, tp_size)
assert results[0][2].max_num_logits_per_req == 1
for local, _, metadata in results:
assert metadata.num_local_logits == metadata.num_local_reqs == local.num_reqs
owned = _owned_batch_indices(local)
assert _np(local.logits_indices).tolist() == (
_np(batch.logits_indices)[owned].tolist()
)
@pytest.mark.parametrize("tp_size", [2, 4])
def test_shard_grammar_output(tp_size: int):
"""The grammar filter keeps exactly the owned requests' bitmask rows, in
grammar order, and the per-rank pieces partition the global bitmask."""
rng = np.random.default_rng(5)
max_num_reqs = 32
num_reqs = 12
slots = rng.choice(np.arange(max_num_reqs), size=num_reqs, replace=False)
k = rng.integers(0, 4, size=num_reqs)
cu = np.zeros(num_reqs + 1, dtype=np.int32)
np.cumsum(1 + k, out=cu[1:])
owner = slots % tp_size
# Grammar requests: every other request, in a scrambled order.
req_ids = [f"req-{i}" for i in range(num_reqs)]
grammar_idx = list(rng.permutation(np.arange(0, num_reqs, 2)))
grammar_req_ids = [req_ids[i] for i in grammar_idx]
num_grammar_rows = sum(int(cu[i + 1] - cu[i]) for i in grammar_idx)
bitmask = rng.integers(0, 2**31, size=(num_grammar_rows, 4), dtype=np.int32)
input_batch = SimpleNamespace(req_ids=req_ids, cu_num_logits_np=cu)
# Global row offset of each grammar request within the bitmask.
row_offsets = {}
cursor = 0
for i in grammar_idx:
row_offsets[i] = cursor
cursor += int(cu[i + 1] - cu[i])
grammar_output = GrammarOutput(
structured_output_request_ids=grammar_req_ids,
grammar_bitmask=bitmask,
)
kept_total = 0
for rank in range(tp_size):
local_req_ids = [req_ids[i] for i in range(num_reqs) if owner[i] == rank]
local = _shard_grammar_output(grammar_output, input_batch, local_req_ids)
expected_idx = [i for i in grammar_idx if owner[i] == rank]
if not expected_idx:
assert local is None
continue
assert local is not None
assert local.structured_output_request_ids == [req_ids[i] for i in expected_idx]
expected_rows = np.concatenate(
[
np.arange(row_offsets[i], row_offsets[i] + cu[i + 1] - cu[i])
for i in expected_idx
]
)
assert (local.grammar_bitmask == bitmask[expected_rows.astype(int)]).all()
kept_total += local.grammar_bitmask.shape[0]
assert kept_total == num_grammar_rows
@requires_cuda
@pytest.mark.parametrize("tp_size", [2, 4])
def test_gather_src_indices_round_trip(tp_size: int):
"""Padded per-rank result blocks indexed by gathered_src_indices restore
batch order, including when some rank owns zero requests."""
rng = np.random.default_rng(3)
max_num_reqs = 40
all_slots = np.arange(max_num_reqs)
batch = _make_batch(
rng,
num_reqs=17,
max_num_reqs=max_num_reqs,
max_spec=2,
slot_pool=all_slots[all_slots % tp_size != 0], # rank 0 owns nothing
)
results = _shard_all_ranks(batch, max_num_reqs, tp_size)
metadata0 = results[0][2]
assert metadata0.num_local_reqs == 0
# Value of request j is j; each rank packs its owned requests' values into
# a zero-padded (max_num_reqs_per_rank, 1) block, gathered rank-major.
pad = metadata0.max_num_reqs_per_rank
blocks = []
for local, _, _ in results:
owned = _owned_batch_indices(local)
block = torch.zeros(pad, 1, dtype=torch.int64, device=DEVICE)
block[: len(owned), 0] = torch.from_numpy(owned).to(DEVICE)
blocks.append(block)
gathered = torch.cat(blocks)
restored = gathered[metadata0.gathered_src_indices][:, 0]
assert restored.tolist() == list(range(batch.num_reqs))
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize("tp_size", [2, 4])
def test_gather_sampler_output_kernels(tp_size: int):
"""The fused pack/unpack kernels round-trip per-rank sampler results
into batch order: token ids zero-filled past the local width, counts
cast back to int32, identical on every rank."""
from dataclasses import replace as dc_replace
from vllm.v1.worker.gpu.sample.output import SamplerOutput
rng = np.random.default_rng(9)
max_num_reqs = 32
batch = _make_batch(rng, num_reqs=13, max_num_reqs=max_num_reqs, max_spec=3)
results = _shard_all_ranks(batch, max_num_reqs, tp_size)
device = torch.device("cuda")
num_reqs = batch.num_reqs
width = results[0][2].max_num_logits_per_req
# Distinctive per-rank local outputs. Even ranks mimic the regular
# sampler (one column, exercising zero-fill); odd ranks mimic the
# rejection sampler's over-allocation (exercising truncation).
local_outputs: list[SamplerOutput | None] = []
expected_ids = torch.zeros(num_reqs, width, dtype=torch.int64)
expected_sampled = torch.zeros(num_reqs, dtype=torch.int32)
expected_rejected = torch.zeros(num_reqs, dtype=torch.int32)
for rank, (local, _, metadata) in enumerate(results):
if metadata.num_local_reqs == 0:
local_outputs.append(None)
continue
owned = torch.from_numpy(_owned_batch_indices(local))
src_width = 1 if rank % 2 == 0 else width + 3
ids = owned[:, None] * 1000 + torch.arange(src_width)[None, :]
num_sampled = (owned % width + 1).to(torch.int32)
num_rejected = (owned % 3).to(torch.int32)
local_outputs.append(
SamplerOutput(
sampled_token_ids=ids.to(device),
logprobs_tensors=None,
num_nans=None,
num_sampled=num_sampled.to(device),
num_rejected=num_rejected.to(device),
)
)
copy_width = min(src_width, width)
expected_ids[owned, :copy_width] = ids[:, :copy_width]
expected_sampled[owned] = num_sampled
expected_rejected[owned] = num_rejected
# Two passes over a mocked all-gather: capture each rank's packed send
# block, then replay with the concatenated blocks as the gathered result.
packed_blocks: dict[int, torch.Tensor] = {}
gathered_full: list[torch.Tensor] = []
def run(rank: int) -> SamplerOutput:
metadata = dc_replace(
results[rank][2],
gathered_src_indices=results[rank][2].gathered_src_indices.to(device),
)
def fake_all_gather(x: torch.Tensor, dim: int = 0) -> torch.Tensor:
packed_blocks[rank] = x.clone()
if gathered_full:
return gathered_full[0]
return torch.zeros(
tp_size * x.shape[0], x.shape[1], dtype=x.dtype, device=x.device
)
with mock.patch.object(
batch_shard, "tensor_model_parallel_all_gather", fake_all_gather
):
return batch_shard.gather_sampler_output(
local_outputs[rank],
metadata,
device,
global_batch=batch,
local_batch=results[rank][0],
)
for rank in range(tp_size):
run(rank)
# Zero the uninitialized padding rows so the comparison is deterministic;
# the real path never reads them (gathered_src_indices skips padding).
for rank, (local, _, metadata) in enumerate(results):
packed_blocks[rank][metadata.num_local_reqs :] = 0
gathered_full.append(torch.cat([packed_blocks[r] for r in range(tp_size)]))
for rank in range(tp_size):
out = run(rank)
assert out.sampled_token_ids.dtype == torch.int64
assert out.num_sampled is not None and out.num_sampled.dtype == torch.int32
assert out.num_rejected is not None
assert torch.equal(out.sampled_token_ids.cpu(), expected_ids)
assert torch.equal(out.num_sampled.cpu(), expected_sampled)
assert torch.equal(out.num_rejected.cpu(), expected_rejected.to(torch.int32))
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_gather_sampler_output_logprobs_and_nans():
"""num_nans is reduced per request and gathered; LogprobsTensors are
packed into fixed per-request blocks, width-aligned across ranks (pad
and truncate paths), and restored to global row order."""
from dataclasses import replace as dc_replace
from vllm.v1.outputs import LogprobsTensors
from vllm.v1.worker.gpu.sample.output import SamplerOutput
tp_size = 2
rng = np.random.default_rng(21)
max_num_reqs = 32
batch = _make_batch(rng, num_reqs=6, max_num_reqs=max_num_reqs, max_spec=3)
results = _shard_all_ranks(batch, max_num_reqs, tp_size)
device = torch.device("cuda")
num_reqs = batch.num_reqs
cu = batch.cu_num_logits_np
num_logits = int(cu[-1])
width = results[0][2].max_num_logits_per_req
num_cols = 3
def cuda_fields(b: InputBatch) -> InputBatch:
return dc_replace(
b,
cu_num_logits=b.cu_num_logits.to(device),
expanded_local_pos=b.expanded_local_pos.to(device),
)
global_cuda = cuda_fields(batch)
local_outputs: list[SamplerOutput | None] = []
local_batches: list[InputBatch] = []
expected_ids = torch.zeros(num_logits, num_cols, dtype=torch.int64)
expected_logprobs = torch.full(
(num_logits, num_cols), float("-inf"), dtype=torch.float32
)
expected_ranks = torch.zeros(num_logits, dtype=torch.int64)
expected_nans = torch.zeros(num_reqs, dtype=torch.int32)
for rank, (local, _, metadata) in enumerate(results):
local_batches.append(cuda_fields(local))
if metadata.num_local_reqs == 0:
local_outputs.append(None)
continue
owned = _owned_batch_indices(local)
rows = torch.from_numpy(_owned_rows(cu, owned))
# Rank 0 sends fewer columns (pad path), rank 1 more (truncate path).
local_cols = num_cols - 1 if rank == 0 else num_cols + 2
ids = rows[:, None] * 10 + torch.arange(local_cols)[None, :]
logprobs = ids.float() * 0.5
ranks_t = rows + 7
nans = (rows % 4).to(torch.int32)
local_outputs.append(
SamplerOutput(
sampled_token_ids=torch.zeros(
metadata.num_local_reqs, width, dtype=torch.int64, device=device
),
logprobs_tensors=LogprobsTensors(
logprob_token_ids=ids.to(device),
logprobs=logprobs.to(device),
selected_token_ranks=ranks_t.to(device),
cu_num_generated_tokens=None,
),
num_nans=nans.to(device),
num_sampled=torch.ones(
metadata.num_local_reqs, dtype=torch.int32, device=device
),
num_rejected=torch.zeros(
metadata.num_local_reqs, dtype=torch.int32, device=device
),
)
)
copy_cols = min(local_cols, num_cols)
expected_ids[rows, :copy_cols] = ids[:, :copy_cols]
expected_logprobs[rows, :copy_cols] = logprobs[:, :copy_cols]
expected_ranks[rows] = ranks_t
for j in owned:
in_req = (rows >= int(cu[j])) & (rows < int(cu[j + 1]))
expected_nans[j] = int(nans[in_req].sum())
recorded: dict[int, list[torch.Tensor]] = {}
gathered_full: list[torch.Tensor] = []
def run(rank: int) -> SamplerOutput:
metadata = dc_replace(
results[rank][2],
gathered_src_indices=results[rank][2].gathered_src_indices.to(device),
)
calls = {"i": 0}
def fake_all_gather(x: torch.Tensor, dim: int = 0) -> torch.Tensor:
i = calls["i"]
calls["i"] += 1
if gathered_full:
return gathered_full[i]
recorded.setdefault(rank, []).append(x.clone())
return torch.zeros(
tp_size * x.shape[0], *x.shape[1:], dtype=x.dtype, device=x.device
)
with mock.patch.object(
batch_shard, "tensor_model_parallel_all_gather", fake_all_gather
):
return batch_shard.gather_sampler_output(
local_outputs[rank],
metadata,
device,
global_batch=global_cuda,
local_batch=local_batches[rank],
gather_num_nans=True,
logprobs_dims=(num_cols - 1, 0),
)
for rank in range(tp_size):
run(rank)
gathered_full.extend(
torch.cat([recorded[r][i] for r in range(tp_size)]) for i in range(3)
)
for rank in range(tp_size):
out = run(rank)
assert out.num_nans is not None
assert torch.equal(out.num_nans.cpu(), expected_nans)
lp = out.logprobs_tensors
assert lp is not None
assert torch.equal(lp.logprob_token_ids.cpu(), expected_ids)
assert torch.equal(lp.logprobs.cpu(), expected_logprobs)
assert torch.equal(lp.selected_token_ranks.cpu(), expected_ranks)
if num_logits != num_reqs:
assert lp.cu_num_generated_tokens == cu.tolist()
else:
assert lp.cu_num_generated_tokens is None
def test_finish_requests_frees_slots_in_sorted_order():
"""Request slots must be freed in the same order on every TP rank:
ownership derives from slot indices, and `finished_req_ids` is a set
whose iteration order is per-process hash-randomized."""
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
removed: list[str] = []
fake_runner = SimpleNamespace(_remove_request=removed.append, pooling_runner=None)
scheduler_output = SimpleNamespace(
finished_req_ids={"req-b", "req-c", "req-a"},
preempted_req_ids={"req-e", "req-d"},
)
GPUModelRunner.finish_requests(fake_runner, scheduler_output)
assert removed == sorted(removed)
assert set(removed) == {"req-a", "req-b", "req-c", "req-d", "req-e"}
@@ -60,6 +60,7 @@ def _make_runner(**overrides: Any) -> Any:
runner.parallel_config = SimpleNamespace(
enable_eplb=True,
enable_elastic_ep=False,
enable_batch_sharded_sampling=False,
eplb_config=SimpleNamespace(log_balancedness=True),
)
runner.vllm_config = SimpleNamespace(
+7
View File
@@ -164,6 +164,13 @@ class ParallelConfig:
"""Whether the deployed model is MoE (if known)."""
enable_expert_parallel: bool = False
"""Use expert parallelism instead of tensor parallelism for MoE layers."""
enable_batch_sharded_sampling: bool | None = None
"""Use sharded sampling across tensor parallel ranks. Each rank samples
a slice of the batch instead of every rank sampling all of it. Currently
defaults to False if not set. Enabling it explicitly raises when the config
cannot support it (`tensor_parallel_size` must be > 1, `max_num_seqs` at
least `tensor_parallel_size`, and `max_logprobs` non-negative). Models opt in
by implementing `compute_logits_local`."""
enable_ep_weight_filter: bool = False
"""Skip non-local expert weights during model loading when expert
parallelism is active. Each rank only reads its own expert shard from
+53
View File
@@ -1675,6 +1675,8 @@ class VllmConfig:
"Remove VLLM_USE_V2_MODEL_RUNNER=0."
)
self._validate_batch_sharded_sampling()
# Re-compute compile ranges after platform-specific config updates
# (e.g., XPU may lower max_num_batched_tokens when MLA is enabled)
self._set_compile_ranges()
@@ -2552,6 +2554,57 @@ class VllmConfig:
return unsupported
def _validate_batch_sharded_sampling(self) -> None:
"""Validate `enable_batch_sharded_sampling` against the rest of the config."""
if not self.parallel_config.enable_batch_sharded_sampling:
# Default to False if not set.
self.parallel_config.enable_batch_sharded_sampling = False
return
blockers: list[str] = []
tp_size = self.parallel_config.tensor_parallel_size
if not self.use_v2_model_runner:
blockers.append("it is only implemented by Model Runner V2")
if tp_size <= 1:
blockers.append("tensor_parallel_size is 1, so there is nothing to shard")
elif self.scheduler_config.max_num_seqs < tp_size:
# Requests are assigned to ranks whole, so fewer slots than ranks
# leaves some ranks without work in every step.
blockers.append(
f"max_num_seqs ({self.scheduler_config.max_num_seqs}) is below "
f"tensor_parallel_size ({tp_size})"
)
if self.model_config is not None and self.model_config.max_logprobs < 0:
# max_logprobs == -1 allows vocab-size logprob requests, which the
# fixed-width logprobs gather cannot reasonably size for.
blockers.append("max_logprobs is -1, allowing vocab-size logprob requests")
if (
self.speculative_config is not None
and self.speculative_config.enable_adaptive_verification
):
# Adaptive verification picks the per-request draft split on the GPU,
# so cu_num_logits_np is only an upper bound, while the shard plan is
# built from that CPU array. The two disagree once the budget binds.
# TODO(TheEpicDolphin): Support adaptive verification with batch-sharded
# sampling.
blockers.append(
"it does not yet work with adaptive verification, which decides "
"the per-request logits counts on the GPU, where the CPU-side "
"shard plan cannot see them"
)
if blockers:
raise ValueError(
"Batch-sharded sampling was explicitly enabled via "
"the --enable-batch-sharded-sampling flag, but is not supported "
"in this configuration for the following reason(s): "
f"{'; '.join(blockers)}."
)
def _validate_v2_model_runner(self) -> None:
"""Check for features not yet supported by the V2 model runner."""
if not HAS_TRITON:
+8
View File
@@ -496,6 +496,9 @@ class EngineArgs:
data_parallel_multi_port_external_lb: bool = False
data_parallel_backend: DataParallelBackend = ParallelConfig.data_parallel_backend
enable_expert_parallel: bool = ParallelConfig.enable_expert_parallel
enable_batch_sharded_sampling: bool | None = (
ParallelConfig.enable_batch_sharded_sampling
)
enable_ep_weight_filter: bool = ParallelConfig.enable_ep_weight_filter
moe_backend: MoEBackend = KernelConfig.moe_backend
linear_backend: LinearBackend = KernelConfig.linear_backend
@@ -1156,6 +1159,10 @@ class EngineArgs:
"-ep",
**parallel_kwargs["enable_expert_parallel"],
)
parallel_group.add_argument(
"--enable-batch-sharded-sampling",
**parallel_kwargs["enable_batch_sharded_sampling"],
)
parallel_group.add_argument(
"--enable-ep-weight-filter",
**parallel_kwargs["enable_ep_weight_filter"],
@@ -2279,6 +2286,7 @@ class EngineArgs:
data_parallel_hybrid_lb=self.data_parallel_hybrid_lb,
is_moe_model=model_config.is_moe,
enable_expert_parallel=self.enable_expert_parallel,
enable_batch_sharded_sampling=self.enable_batch_sharded_sampling,
enable_ep_weight_filter=self.enable_ep_weight_filter,
all2all_backend=self.all2all_backend,
enable_elastic_ep=self.enable_elastic_ep,
+7
View File
@@ -155,7 +155,14 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
hidden_states: torch.Tensor,
lm_head: VocabParallelEmbedding,
embedding_bias: torch.Tensor | None = None,
skip_gather: bool = False,
) -> torch.Tensor | None:
# The LoRA delta is accumulated into the full gathered logits, so the
# TP gather cannot be skipped here.
if skip_gather:
raise NotImplementedError(
"Skipping the logits TP gather is not supported with an lm_head LoRA."
)
# Get the logits for the next tokens.
if hasattr(lm_head, "base_layer"):
actual_lm_head = lm_head.base_layer
@@ -99,12 +99,15 @@ class LogitsProcessor(PluggableLayer):
lm_head: VocabParallelEmbedding,
hidden_states: torch.Tensor,
embedding_bias: torch.Tensor | None = None,
skip_gather: bool = False,
) -> torch.Tensor | None:
if self.logits_as_input:
logits = hidden_states
else:
# Get the logits for the next tokens.
logits = self._get_logits(hidden_states, lm_head, embedding_bias)
logits = self._get_logits(
hidden_states, lm_head, embedding_bias, skip_gather
)
if logits is not None:
if self.soft_cap is not None:
logits = logits / self.soft_cap
@@ -173,9 +176,12 @@ class LogitsProcessor(PluggableLayer):
hidden_states: torch.Tensor,
lm_head: VocabParallelEmbedding,
embedding_bias: torch.Tensor | None,
skip_gather: bool = False,
) -> torch.Tensor | None:
# Get the logits for the next tokens.
logits = self._apply_head(lm_head, hidden_states, embedding_bias)
if skip_gather:
return logits
# Gather logits for TP
if lm_head.tp_size > 1:
+6
View File
@@ -532,6 +532,12 @@ class LlamaForCausalLM(
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
def compute_logits_local(
self,
hidden_states: torch.Tensor,
) -> torch.Tensor:
return self.logits_processor(self.lm_head, hidden_states, skip_gather=True)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
+9
View File
@@ -429,6 +429,12 @@ class Qwen3_5ForCausalLMBase(
) -> torch.Tensor | None:
return self.logits_processor(self.lm_head, hidden_states)
def compute_logits_local(
self,
hidden_states: torch.Tensor,
) -> torch.Tensor:
return self.logits_processor(self.lm_head, hidden_states, skip_gather=True)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
@@ -591,6 +597,9 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid)
return hidden_states
def compute_logits_local(self, hidden_states: torch.Tensor) -> torch.Tensor:
return self.language_model.compute_logits_local(hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+6
View File
@@ -1765,6 +1765,12 @@ class DeepseekV4ForCausalLM(
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
def compute_logits_local(
self,
hidden_states: torch.Tensor,
) -> torch.Tensor:
return self.logits_processor(self.lm_head, hidden_states, skip_gather=True)
def forward(
self,
input_ids: torch.Tensor,
+11 -4
View File
@@ -63,6 +63,7 @@ class InklingLogitsProcessor(LogitsProcessor):
lm_head: VocabParallelEmbedding,
hidden_states: torch.Tensor,
embedding_bias: torch.Tensor | None = None,
skip_gather: bool = False,
) -> torch.Tensor | None:
# ``base_layer`` exists only on the LogitsProcessorWithLoRA wrapper,
# which calls this forward with ``self`` bound to the wrapper. The
@@ -72,15 +73,16 @@ class InklingLogitsProcessor(LogitsProcessor):
# lives on this class).
if hasattr(self, "base_layer"):
return type(self.base_layer)._lora_forward(
self, lm_head, hidden_states, embedding_bias
self, lm_head, hidden_states, embedding_bias, skip_gather
)
return self._base_forward(lm_head, hidden_states, embedding_bias)
return self._base_forward(lm_head, hidden_states, embedding_bias, skip_gather)
def _lora_forward(
self,
lm_head: VocabParallelEmbedding,
hidden_states: torch.Tensor,
embedding_bias: torch.Tensor | None = None,
skip_gather: bool = False,
) -> torch.Tensor | None:
# ``self`` is the LogitsProcessorWithLoRA wrapper here: ``_get_logits``
# returns the base logits plus the lm_head LoRA delta. Apply the muP
@@ -90,7 +92,9 @@ class InklingLogitsProcessor(LogitsProcessor):
if self.logits_as_input:
logits = hidden_states
else:
logits = self._get_logits(hidden_states, lm_head, embedding_bias)
logits = self._get_logits(
hidden_states, lm_head, embedding_bias, skip_gather
)
# TODO: fuse this multiplication
if logits is not None and mup:
assert self.base_layer.soft_cap is None
@@ -103,10 +107,11 @@ class InklingLogitsProcessor(LogitsProcessor):
lm_head: VocabParallelEmbedding,
hidden_states: torch.Tensor,
embedding_bias: torch.Tensor | None = None,
skip_gather: bool = False,
) -> torch.Tensor | None:
mup = self.logits_mup_width_multiplier
if not mup:
return super().forward(lm_head, hidden_states, embedding_bias)
return super().forward(lm_head, hidden_states, embedding_bias, skip_gather)
# Fold the muP width divisor into the lm_head GEMM alpha (fp32 epilogue):
# no separate elementwise kernel, no bf16 rounding of scaled logits, and
# no weight mutation. Overfit to the served checkpoint: bf16 lm_head, no
@@ -123,6 +128,8 @@ class InklingLogitsProcessor(LogitsProcessor):
beta=0.0,
alpha=1.0 / mup,
)
if skip_gather:
return logits
logits = self._gather_logits(logits)
if logits is not None:
logits = logits[..., : self.org_vocab_size]
+11 -4
View File
@@ -63,6 +63,7 @@ class InklingLogitsProcessor(LogitsProcessor):
lm_head: VocabParallelEmbedding,
hidden_states: torch.Tensor,
embedding_bias: torch.Tensor | None = None,
skip_gather: bool = False,
) -> torch.Tensor | None:
# ``base_layer`` exists only on the LogitsProcessorWithLoRA wrapper,
# which calls this forward with ``self`` bound to the wrapper. The
@@ -72,15 +73,16 @@ class InklingLogitsProcessor(LogitsProcessor):
# lives on this class).
if hasattr(self, "base_layer"):
return type(self.base_layer)._lora_forward(
self, lm_head, hidden_states, embedding_bias
self, lm_head, hidden_states, embedding_bias, skip_gather
)
return self._base_forward(lm_head, hidden_states, embedding_bias)
return self._base_forward(lm_head, hidden_states, embedding_bias, skip_gather)
def _lora_forward(
self,
lm_head: VocabParallelEmbedding,
hidden_states: torch.Tensor,
embedding_bias: torch.Tensor | None = None,
skip_gather: bool = False,
) -> torch.Tensor | None:
# ``self`` is the LogitsProcessorWithLoRA wrapper here: ``_get_logits``
# returns the base logits plus the lm_head LoRA delta. Apply the muP
@@ -90,7 +92,9 @@ class InklingLogitsProcessor(LogitsProcessor):
if self.logits_as_input:
logits = hidden_states
else:
logits = self._get_logits(hidden_states, lm_head, embedding_bias)
logits = self._get_logits(
hidden_states, lm_head, embedding_bias, skip_gather
)
# TODO: fuse this multiplication
if logits is not None and mup:
assert self.base_layer.soft_cap is None
@@ -103,10 +107,11 @@ class InklingLogitsProcessor(LogitsProcessor):
lm_head: VocabParallelEmbedding,
hidden_states: torch.Tensor,
embedding_bias: torch.Tensor | None = None,
skip_gather: bool = False,
) -> torch.Tensor | None:
mup = self.logits_mup_width_multiplier
if not mup:
return super().forward(lm_head, hidden_states, embedding_bias)
return super().forward(lm_head, hidden_states, embedding_bias, skip_gather)
# Fold the muP width divisor into the lm_head GEMM alpha (fp32 epilogue):
# no separate elementwise kernel, no bf16 rounding of scaled logits, and
# no weight mutation. Overfit to the served checkpoint: bf16 lm_head, no
@@ -123,6 +128,8 @@ class InklingLogitsProcessor(LogitsProcessor):
beta=0.0,
alpha=1.0 / mup,
)
if skip_gather:
return logits
logits = self._gather_logits(logits)
if logits is not None:
logits = logits[..., : self.org_vocab_size]
+6
View File
@@ -1059,6 +1059,9 @@ class MiniMaxM3SparseForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
return self.logits_processor(self.lm_head, hidden_states)
def compute_logits_local(self, hidden_states: torch.Tensor) -> torch.Tensor:
return self.logits_processor(self.lm_head, hidden_states, skip_gather=True)
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
return self.model.get_expert_mapping()
@@ -1272,6 +1275,9 @@ class MiniMaxM3SparseForConditionalGeneration(
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
return self.language_model.compute_logits(hidden_states)
def compute_logits_local(self, hidden_states: torch.Tensor) -> torch.Tensor:
return self.language_model.compute_logits_local(hidden_states)
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
return self.language_model.get_expert_mapping()
+72 -4
View File
@@ -127,6 +127,11 @@ from vllm.v1.worker.gpu.mm.lora import set_active_mm_loras
from vllm.v1.worker.gpu.model_states import init_model_state
from vllm.v1.worker.gpu.pool.pooling_runner import PoolingRunner
from vllm.v1.worker.gpu.pp_utils import PPHandler
from vllm.v1.worker.gpu.sample.batch_shard import (
BatchSharder,
all_to_all_logits,
gather_sampler_output,
)
from vllm.v1.worker.gpu.sample.output import SamplerOutput
from vllm.v1.worker.gpu.sample.prompt_logprob import PromptLogprobsWorker
from vllm.v1.worker.gpu.sample.sampler import Sampler
@@ -310,6 +315,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# model_state exists (num_new_sampled_tokens_per_step from ModelState).
self.sampler: Sampler | None = None
self.rejection_sampler: RejectionSampler | None = None
self.batch_sharder: BatchSharder | None = None
self.prompt_logprobs_worker: PromptLogprobsWorker | None = None
self.structured_outputs_worker: StructuredOutputsWorker | None = None
self.cudagraph_manager: ModelCudaGraphManager | None = None
@@ -404,6 +410,21 @@ class GPUModelRunner(LoRAModelRunnerMixin):
+ self.model_state.num_new_sampled_tokens_per_step
)
if self.parallel_config.enable_batch_sharded_sampling:
if hasattr(self.model, "compute_logits_local"):
self.batch_sharder = BatchSharder(
max_num_reqs=self.max_num_reqs,
max_num_logits_per_req=self.decode_query_len,
device=self.device,
)
logger.info("Batch-sharded sampling enabled.")
else:
logger.warning_once(
"Disabling batch-sharded sampling: %s does not implement "
"compute_logits_local",
type(self.model).__name__,
)
# Initialize samplers. Model states may override via custom_sampler().
if self.is_last_pp_rank and not self.is_pooling_model:
self.sampler = Sampler(
@@ -935,7 +956,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
preempted_req_ids = scheduler_output.preempted_req_ids
if preempted_req_ids:
finished_req_ids = finished_req_ids.union(preempted_req_ids)
for req_id in finished_req_ids:
# Sorted so every TP rank frees request slots in the same order.
# Features like batch-sharded sampling derive rank request ownership
# from the slot index.
for req_id in sorted(finished_req_ids):
self._remove_request(req_id)
def free_states(self, scheduler_output: SchedulerOutput) -> None:
@@ -1335,8 +1359,25 @@ class GPUModelRunner(LoRAModelRunnerMixin):
input_batch: InputBatch,
grammar_output: GrammarOutput | None,
) -> tuple[SamplerOutput, torch.Tensor, torch.Tensor]:
sample_hidden_states = hidden_states[input_batch.logits_indices]
logits = self.model.compute_logits(sample_hidden_states)
shard_metadata = None
global_input_batch = input_batch
if self.batch_sharder is not None:
# Shard the inputs along the batch dimension to sample in parallel
# across TP ranks.
input_batch, sorted_logits_indices, grammar_output, shard_metadata = (
self.batch_sharder.shard_sampler_inputs(input_batch, grammar_output)
)
# The hidden states must be gathered in rank-owner-sorted order
# before computing the partial-vocab logits, so that the all-to-all
# produces full-vocab logits for just the locally-owned requests.
sample_hidden_states = hidden_states[sorted_logits_indices]
local_logits = self.model.compute_logits_local(sample_hidden_states)
logits = all_to_all_logits(local_logits, shard_metadata)
logits = logits[:, : self.vocab_size]
else:
sample_hidden_states = hidden_states[input_batch.logits_indices]
logits = self.model.compute_logits(sample_hidden_states)
if grammar_output is not None:
# Apply grammar bitmask to the logits in-place.
assert self.structured_outputs_worker is not None
@@ -1347,7 +1388,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
grammar_output.grammar_bitmask,
)
if input_batch.num_draft_tokens == 0 or self.rejection_sampler is None:
sampler_output: SamplerOutput | None
if input_batch.num_reqs == 0:
# This rank owns no requests this step. It contributes an
# all-padding block to the gather below.
sampler_output = None
elif input_batch.num_draft_tokens == 0 or self.rejection_sampler is None:
assert self.sampler is not None
sampler_output = self.sampler(logits, input_batch)
else:
@@ -1361,6 +1407,28 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.speculator.draft_logits,
)
if shard_metadata is not None:
# Gather the sharded sampler outputs from the TP ranks into a single
# sampler output.
assert self.sampler is not None
sampler_output = gather_sampler_output(
sampler_output,
shard_metadata,
device=self.device,
global_batch=global_input_batch,
local_batch=input_batch,
gather_num_nans=self.sampler.compute_nans,
logprobs_dims=self.sampler.get_logprobs_dims(
global_input_batch.idx_mapping_np,
# Rejection sampler does not return logprob token ids.
include_token_ids=(
global_input_batch.num_draft_tokens == 0
or self.rejection_sampler is None
),
),
)
assert sampler_output is not None
return sampler_output, sampler_output.num_sampled, sampler_output.num_rejected
def postprocess_sampled(
+674
View File
@@ -0,0 +1,674 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass, replace
import numpy as np
import torch
import torch.distributed
from vllm.distributed.communication_op import tensor_model_parallel_all_gather
from vllm.distributed.parallel_state import get_tp_group
from vllm.triton_utils import tl, triton
from vllm.v1.core.sched.output import GrammarOutput
from vllm.v1.outputs import LogprobsTensors
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.sample.output import SamplerOutput
@dataclass
class BatchShardMetadata:
"""Collective layout for one sampling step.
Requests (and their logits) are owner-sorted: all requests owned by
rank 0 first, then rank 1, etc. Within an owner, batch order is
preserved (stable sort). Every field is a pure function of the
replicated idx_mapping and cu_num_logits, so all ranks build identical
plans without communication — which relies on request slots
(idx_mapping) being assigned identically on every rank (see
finish_requests in the model runner).
"""
tp_size: int
# Per-owner-rank logits counts (all-to-all send splits).
num_logits_per_rank: list[int]
num_local_logits: int
num_local_reqs: int
# The number of request entries contributed by each rank to the
# gathered result (shorter shards are padded up to this value).
max_num_reqs_per_rank: int
# [num_reqs] For batch request i, its source index in the rank-major
# gathered results. Indexing the gathered tensors with this restores
# the original batch order.
gathered_src_indices: torch.Tensor
# The number of entries occupied by each request in the gathered
# tensor. Derived from the global cu_num_logits, so each rank agrees
# ont he gather shapes, including empty shards.
max_num_logits_per_req: int
@triton.jit(
do_not_specialize=["num_reqs", "local_logits_start", "max_num_reqs_per_rank"]
)
def _build_shard_plan_kernel(
idx_mapping_ptr,
cu_num_logits_ptr,
query_start_loc_ptr,
seq_lens_ptr,
sorted_logits_indices_ptr,
gathered_src_indices_ptr,
local_idx_mapping_ptr,
local_cu_num_logits_ptr,
local_logits_indices_ptr,
local_expanded_idx_mapping_ptr,
local_expanded_local_pos_ptr,
local_seq_lens_ptr,
num_reqs,
local_logits_start,
max_num_reqs_per_rank,
TP_SIZE: tl.constexpr,
TP_RANK: tl.constexpr,
PADDED_NUM_REQS: tl.constexpr,
PADDED_NUM_LOGITS_PER_REQ: tl.constexpr,
):
req_idx = tl.program_id(0)
req_state_idx = tl.load(idx_mapping_ptr + req_idx)
owner = req_state_idx % TP_SIZE
num_logits = tl.load(cu_num_logits_ptr + req_idx + 1) - tl.load(
cu_num_logits_ptr + req_idx
)
req_block = tl.arange(0, PADDED_NUM_REQS)
req_mask = req_block < num_reqs
owners = tl.load(idx_mapping_ptr + req_block, mask=req_mask, other=0) % TP_SIZE
num_logits_per_req = tl.load(
cu_num_logits_ptr + req_block + 1, mask=req_mask, other=0
) - tl.load(cu_num_logits_ptr + req_block, mask=req_mask, other=0)
# Owner-sorted layout is rank 0's requests in batch order, then rank 1's,
# etc, so a request's starting offset can be derived from counting.
lower = (owners < owner) & req_mask
earlier = (owners == owner) & (req_block < req_idx) & req_mask
# Compute the logits row offset for the current request in the sorted
# batch.
req_logits_start = tl.sum(tl.where(lower, num_logits_per_req, 0)) + tl.sum(
tl.where(earlier, num_logits_per_req, 0)
)
# Copy the current request logit indices into the sorted batch.
query_start = tl.load(query_start_loc_ptr + req_idx + 1) - num_logits
logit_block = tl.arange(0, PADDED_NUM_LOGITS_PER_REQ)
logit_mask = logit_block < num_logits
logits_indices = (query_start + logit_block).to(tl.int64)
tl.store(
sorted_logits_indices_ptr + req_logits_start + logit_block,
logits_indices,
mask=logit_mask,
)
# Write the mapping of output tensor request indices => gathered tensor
# request indices.
# Batch request i's result lives in the padded, gathered results at
# rank_of(i) * max_num_reqs_per_rank + offset_within_rank(i).
local_req_idx = tl.sum(earlier.to(tl.int32))
tl.store(
gathered_src_indices_ptr + req_idx,
(owner * max_num_reqs_per_rank + local_req_idx).to(tl.int64),
)
if req_idx == 0:
tl.store(local_cu_num_logits_ptr, 0)
if owner == TP_RANK:
local_start = req_logits_start - local_logits_start
tl.store(local_idx_mapping_ptr + local_req_idx, req_state_idx)
tl.store(local_cu_num_logits_ptr + local_req_idx + 1, local_start + num_logits)
tl.store(
local_logits_indices_ptr + local_start + logit_block,
logits_indices,
mask=logit_mask,
)
tl.store(
local_expanded_idx_mapping_ptr + local_start + logit_block,
req_state_idx,
mask=logit_mask,
)
tl.store(
local_expanded_local_pos_ptr + local_start + logit_block,
logit_block,
mask=logit_mask,
)
tl.store(local_seq_lens_ptr + local_req_idx, tl.load(seq_lens_ptr + req_idx))
class BatchSharder:
"""Shards the sampler inputs across TP ranks along the batch dimension."""
def __init__(
self,
max_num_reqs: int,
max_num_logits_per_req: int,
device: torch.device,
):
tp_group = get_tp_group()
self.tp_rank = tp_group.rank_in_group
self.tp_size = tp_group.world_size
self.device = device
self._padded_num_reqs = triton.next_power_of_2(max_num_reqs)
self._padded_num_logits_per_req = triton.next_power_of_2(max_num_logits_per_req)
self._num_warps = max(1, min(8, self._padded_num_reqs // 128))
def shard_sampler_inputs(
self,
input_batch: InputBatch,
grammar_output: GrammarOutput | None,
) -> tuple[InputBatch, torch.Tensor, GrammarOutput | None, BatchShardMetadata]:
"""Owner-sort the batch and build this rank's local sampler inputs.
Returns the local sub-batch, the owner-sorted logits_indices (gather
the sampling hidden states with these so `compute_logits_local` emits
logits in all-to-all send order), the local grammar output (None if
this rank owns none of the structured-output requests), and the shard
metadata (used for all-gathering the sampler outputs).
"""
tp_rank = self.tp_rank
tp_size = self.tp_size
num_reqs = input_batch.idx_mapping_np.shape[0]
num_logits = int(input_batch.cu_num_logits_np[-1])
# Deterministically assign requests to ranks, round-robin over slot
# indices so ownership stays balanced when the slot allocator fills
# low slots first (partial occupancy).
# NOTE: This mirrors the assignment used in _build_shard_plan_kernel.
req_owner_np = input_batch.idx_mapping_np % tp_size
local_req_indices_np = np.flatnonzero(req_owner_np == tp_rank)
local_idx_mapping_np = input_batch.idx_mapping_np[local_req_indices_np]
num_local_reqs = local_req_indices_np.shape[0]
num_reqs_per_rank_np = np.bincount(req_owner_np, minlength=tp_size)
max_num_reqs_per_rank = int(num_reqs_per_rank_np.max()) if num_reqs else 1
# Derive the number of logits owned by each rank, as well as the
# local rank.
num_logits_per_req_np = np.diff(input_batch.cu_num_logits_np)
num_logits_per_rank_np = np.bincount(
req_owner_np, weights=num_logits_per_req_np, minlength=tp_size
).astype(np.int64)
num_local_logits = int(num_logits_per_rank_np[tp_rank])
local_logits_start = int(num_logits_per_rank_np[:tp_rank].sum())
local_cu_num_logits_np = np.zeros(num_local_reqs + 1, dtype=np.int32)
np.cumsum(
num_logits_per_req_np[local_req_indices_np], out=local_cu_num_logits_np[1:]
)
max_num_logits_per_req = int(num_logits_per_req_np.max()) if num_reqs else 1
# Shard the input batch GPU tensors.
sorted_logits_indices = torch.empty(
num_logits, dtype=torch.int64, device=self.device
)
gathered_src_indices = torch.empty(
num_reqs, dtype=torch.int64, device=self.device
)
local_logits_indices = torch.empty(
num_local_logits, dtype=torch.int64, device=self.device
)
local_idx_mapping = torch.empty(
num_local_reqs, dtype=torch.int32, device=self.device
)
local_cu_num_logits = torch.empty(
num_local_reqs + 1, dtype=torch.int32, device=self.device
)
local_expanded_idx_mapping = torch.empty(
num_local_logits, dtype=torch.int32, device=self.device
)
local_expanded_local_pos = torch.empty(
num_local_logits, dtype=torch.int32, device=self.device
)
local_seq_lens = torch.empty(
num_local_reqs, dtype=torch.int32, device=self.device
)
if num_reqs > 0:
_build_shard_plan_kernel[(num_reqs,)](
input_batch.idx_mapping,
input_batch.cu_num_logits,
input_batch.query_start_loc,
input_batch.seq_lens,
sorted_logits_indices,
gathered_src_indices,
local_idx_mapping,
local_cu_num_logits,
local_logits_indices,
local_expanded_idx_mapping,
local_expanded_local_pos,
local_seq_lens,
num_reqs,
local_logits_start,
max_num_reqs_per_rank,
TP_SIZE=tp_size,
TP_RANK=tp_rank,
PADDED_NUM_REQS=self._padded_num_reqs,
PADDED_NUM_LOGITS_PER_REQ=self._padded_num_logits_per_req,
num_warps=self._num_warps,
)
# Compute the local number of draft tokens.
num_draft_tokens_per_req = None
num_draft_tokens = 0
if input_batch.num_draft_tokens_per_req is not None:
num_draft_tokens_per_req = input_batch.num_draft_tokens_per_req[
local_req_indices_np
]
num_draft_tokens = int(num_draft_tokens_per_req.sum())
local_req_ids = [input_batch.req_ids[i] for i in local_req_indices_np.tolist()]
local_batch = replace(
input_batch,
req_ids=local_req_ids,
num_reqs=num_local_reqs,
idx_mapping=local_idx_mapping,
idx_mapping_np=local_idx_mapping_np,
expanded_idx_mapping=local_expanded_idx_mapping,
expanded_local_pos=local_expanded_local_pos,
seq_lens=local_seq_lens,
logits_indices=local_logits_indices,
cu_num_logits=local_cu_num_logits,
cu_num_logits_np=local_cu_num_logits_np,
num_draft_tokens=num_draft_tokens,
num_draft_tokens_per_req=num_draft_tokens_per_req,
)
local_grammar_output = None
if grammar_output is not None:
local_grammar_output = _shard_grammar_output(
grammar_output, input_batch, local_req_ids
)
metadata = BatchShardMetadata(
tp_size=tp_size,
num_logits_per_rank=num_logits_per_rank_np.tolist(),
num_local_logits=num_local_logits,
num_local_reqs=num_local_reqs,
max_num_reqs_per_rank=max_num_reqs_per_rank,
gathered_src_indices=gathered_src_indices,
max_num_logits_per_req=max_num_logits_per_req,
)
return local_batch, sorted_logits_indices, local_grammar_output, metadata
def all_to_all_logits(
logits_shard: torch.Tensor,
metadata: BatchShardMetadata,
) -> torch.Tensor:
recv = logits_shard.new_empty(
metadata.tp_size * metadata.num_local_logits, logits_shard.shape[-1]
)
torch.distributed.all_to_all_single(
recv,
logits_shard.contiguous(),
output_split_sizes=[metadata.num_local_logits] * metadata.tp_size,
input_split_sizes=metadata.num_logits_per_rank,
group=get_tp_group().device_group,
)
shard_width = recv.shape[-1]
return (
recv.view(metadata.tp_size, metadata.num_local_logits, shard_width)
.permute(1, 0, 2)
.reshape(metadata.num_local_logits, metadata.tp_size * shard_width)
)
def _shard_grammar_output(
grammar_output: GrammarOutput,
input_batch: InputBatch,
local_req_ids: list[str],
) -> GrammarOutput | None:
owned = set(local_req_ids)
req_id_to_idx = {req_id: i for i, req_id in enumerate(input_batch.req_ids)}
cu_num_logits_np = input_batch.cu_num_logits_np
local_ids: list[str] = []
keep_indices: list[int] = []
cursor = 0
for req_id in grammar_output.structured_output_request_ids:
req_idx = req_id_to_idx[req_id]
num_req_logits = int(cu_num_logits_np[req_idx + 1] - cu_num_logits_np[req_idx])
if req_id in owned:
local_ids.append(req_id)
keep_indices.extend(range(cursor, cursor + num_req_logits))
cursor += num_req_logits
if not local_ids:
return None
return GrammarOutput(
structured_output_request_ids=local_ids,
grammar_bitmask=grammar_output.grammar_bitmask[keep_indices],
)
@triton.jit(
do_not_specialize=["max_num_logits_per_req", "num_src_cols", "packed_stride"]
)
def _pack_sampler_output_kernel(
packed_ptr,
packed_stride,
sampled_token_ids_ptr,
sampled_token_ids_stride,
num_sampled_ptr,
num_rejected_ptr,
num_nans_ptr,
local_cu_num_logits_ptr,
max_num_logits_per_req,
num_src_cols,
BLOCK_SIZE: tl.constexpr,
):
req_idx = tl.program_id(0)
cols = tl.arange(0, BLOCK_SIZE)
row_ptr = packed_ptr + req_idx * packed_stride
token_ids = tl.load(
sampled_token_ids_ptr + req_idx * sampled_token_ids_stride + cols,
mask=cols < num_src_cols,
other=0,
)
tl.store(row_ptr + cols, token_ids, mask=cols < max_num_logits_per_req)
num_sampled = tl.load(num_sampled_ptr + req_idx)
tl.store(row_ptr + max_num_logits_per_req, num_sampled.to(tl.int64))
num_rejected = tl.load(num_rejected_ptr + req_idx)
tl.store(row_ptr + max_num_logits_per_req + 1, num_rejected.to(tl.int64))
if num_nans_ptr is not None:
# num_nans is per logits row; reduce it per request. The replicated
# samplers report per-row counts that downstream zips per request,
# so a per-request sum is the well-defined equivalent.
start = tl.load(local_cu_num_logits_ptr + req_idx)
num_req_logits = tl.load(local_cu_num_logits_ptr + req_idx + 1) - start
nans = tl.load(num_nans_ptr + start + cols, mask=cols < num_req_logits, other=0)
tl.store(
row_ptr + max_num_logits_per_req + 2, tl.sum(nans.to(tl.int64), axis=0)
)
@triton.jit(do_not_specialize=["max_num_logits_per_req", "gathered_stride"])
def _unpack_gathered_output_kernel(
gathered_ptr,
gathered_stride,
gathered_src_indices_ptr,
sampled_token_ids_ptr,
num_sampled_ptr,
num_rejected_ptr,
num_nans_ptr,
max_num_logits_per_req,
BLOCK_SIZE: tl.constexpr,
):
req_idx = tl.program_id(0)
src = tl.load(gathered_src_indices_ptr + req_idx)
row_ptr = gathered_ptr + src * gathered_stride
cols = tl.arange(0, BLOCK_SIZE)
mask = cols < max_num_logits_per_req
token_ids = tl.load(row_ptr + cols, mask=mask)
tl.store(
sampled_token_ids_ptr + req_idx * max_num_logits_per_req + cols,
token_ids,
mask=mask,
)
num_sampled = tl.load(row_ptr + max_num_logits_per_req)
tl.store(num_sampled_ptr + req_idx, num_sampled.to(tl.int32))
num_rejected = tl.load(row_ptr + max_num_logits_per_req + 1)
tl.store(num_rejected_ptr + req_idx, num_rejected.to(tl.int32))
if num_nans_ptr is not None:
num_nans = tl.load(row_ptr + max_num_logits_per_req + 2)
tl.store(num_nans_ptr + req_idx, num_nans.to(tl.int32))
@triton.jit(
do_not_specialize=[
"max_num_logits_per_req",
"num_logprob_cols",
"num_src_cols",
"send_ids_stride",
"send_logprobs_stride",
"src_ids_stride",
"src_logprobs_stride",
]
)
def _pack_logprobs_kernel(
send_ids_ptr,
send_ids_stride,
send_logprobs_ptr,
send_logprobs_stride,
src_ids_ptr,
src_ids_stride,
src_logprobs_ptr,
src_logprobs_stride,
selected_token_ranks_ptr,
local_cu_num_logits_ptr,
max_num_logits_per_req,
num_logprob_cols,
num_src_cols,
BLOCK_SIZE: tl.constexpr,
):
req_idx = tl.program_id(0)
pos = tl.program_id(1)
start = tl.load(local_cu_num_logits_ptr + req_idx)
if pos < tl.load(local_cu_num_logits_ptr + req_idx + 1) - start:
src = start + pos
dst = req_idx * max_num_logits_per_req + pos
cols = tl.arange(0, BLOCK_SIZE)
# A rank whose sub-batch has no draft tokens runs the regular sampler
# while others run the rejection sampler, and their column counts can
# differ (e.g. logprob_token_ids widening); the two masks pad or
# truncate to the agreed width.
src_mask = cols < num_src_cols
dst_mask = cols < num_logprob_cols
ids = tl.load(src_ids_ptr + src * src_ids_stride + cols, mask=src_mask, other=0)
logprobs = tl.load(
src_logprobs_ptr + src * src_logprobs_stride + cols,
mask=src_mask,
other=float("-inf"),
)
ids_row = send_ids_ptr + dst * send_ids_stride
tl.store(ids_row + cols, ids.to(tl.int64), mask=dst_mask)
rank = tl.load(selected_token_ranks_ptr + src)
tl.store(ids_row + num_logprob_cols, rank.to(tl.int64))
tl.store(
send_logprobs_ptr + dst * send_logprobs_stride + cols,
logprobs.to(tl.float32),
mask=dst_mask,
)
@triton.jit(
do_not_specialize=[
"max_num_logits_per_req",
"num_logprob_cols",
"gathered_ids_stride",
"gathered_logprobs_stride",
]
)
def _unpack_logprobs_kernel(
gathered_ids_ptr,
gathered_ids_stride,
gathered_logprobs_ptr,
gathered_logprobs_stride,
gathered_src_indices_ptr,
cu_num_logits_ptr,
logprob_token_ids_ptr,
logprobs_ptr,
selected_token_ranks_ptr,
max_num_logits_per_req,
num_logprob_cols,
BLOCK_SIZE: tl.constexpr,
):
req_idx = tl.program_id(0)
pos = tl.program_id(1)
start = tl.load(cu_num_logits_ptr + req_idx)
if pos < tl.load(cu_num_logits_ptr + req_idx + 1) - start:
src = tl.load(gathered_src_indices_ptr + req_idx) * max_num_logits_per_req + pos
dst = start + pos
cols = tl.arange(0, BLOCK_SIZE)
mask = cols < num_logprob_cols
src_ids_row = gathered_ids_ptr + src * gathered_ids_stride
ids = tl.load(src_ids_row + cols, mask=mask)
tl.store(logprob_token_ids_ptr + dst * num_logprob_cols + cols, ids, mask=mask)
rank = tl.load(src_ids_row + num_logprob_cols)
tl.store(selected_token_ranks_ptr + dst, rank)
logprobs = tl.load(
gathered_logprobs_ptr + src * gathered_logprobs_stride + cols, mask=mask
)
tl.store(logprobs_ptr + dst * num_logprob_cols + cols, logprobs, mask=mask)
def _gather_logprobs_tensors(
local_output: SamplerOutput | None,
metadata: BatchShardMetadata,
global_batch: InputBatch,
local_batch: InputBatch,
logprobs_dims: tuple[int, int],
device: torch.device,
) -> LogprobsTensors:
num_logprobs, max_token_ids = logprobs_dims
num_logprob_cols = 1 + max(num_logprobs, max_token_ids)
max_num_logits_per_req = metadata.max_num_logits_per_req
num_send_rows = metadata.max_num_reqs_per_rank * max_num_logits_per_req
block_size = triton.next_power_of_2(num_logprob_cols)
# Padded slots are never read back: gathered_src_indices only addresses
# rows that an owner rank wrote.
send_ids_ranks = torch.empty(
num_send_rows, num_logprob_cols + 1, dtype=torch.int64, device=device
)
send_logprobs = torch.empty(
num_send_rows, num_logprob_cols, dtype=torch.float32, device=device
)
if local_output is not None and local_output.logprobs_tensors is not None:
lp = local_output.logprobs_tensors
_pack_logprobs_kernel[(metadata.num_local_reqs, max_num_logits_per_req)](
send_ids_ranks,
send_ids_ranks.stride(0),
send_logprobs,
send_logprobs.stride(0),
lp.logprob_token_ids,
lp.logprob_token_ids.stride(0),
lp.logprobs,
lp.logprobs.stride(0),
lp.selected_token_ranks,
local_batch.cu_num_logits,
max_num_logits_per_req,
num_logprob_cols,
lp.logprob_token_ids.shape[1],
BLOCK_SIZE=block_size,
)
gathered_ids_ranks = tensor_model_parallel_all_gather(send_ids_ranks, dim=0)
gathered_logprobs = tensor_model_parallel_all_gather(send_logprobs, dim=0)
num_reqs = global_batch.num_reqs
num_logits = int(global_batch.cu_num_logits_np[-1])
logprob_token_ids = torch.empty(
num_logits, num_logprob_cols, dtype=torch.int64, device=device
)
logprobs = torch.empty(
num_logits, num_logprob_cols, dtype=torch.float32, device=device
)
selected_token_ranks = torch.empty(num_logits, dtype=torch.int64, device=device)
_unpack_logprobs_kernel[(num_reqs, max_num_logits_per_req)](
gathered_ids_ranks,
gathered_ids_ranks.stride(0),
gathered_logprobs,
gathered_logprobs.stride(0),
metadata.gathered_src_indices,
global_batch.cu_num_logits,
logprob_token_ids,
logprobs,
selected_token_ranks,
max_num_logits_per_req,
num_logprob_cols,
BLOCK_SIZE=block_size,
)
return LogprobsTensors(
logprob_token_ids=logprob_token_ids,
logprobs=logprobs,
selected_token_ranks=selected_token_ranks,
cu_num_generated_tokens=(
global_batch.cu_num_logits_np.tolist() if num_logits != num_reqs else None
),
)
def gather_sampler_output(
local_output: SamplerOutput | None,
metadata: BatchShardMetadata,
device: torch.device,
global_batch: InputBatch,
local_batch: InputBatch,
gather_num_nans: bool = False,
logprobs_dims: tuple[int, int] | None = None,
) -> SamplerOutput:
max_num_logits_per_req = metadata.max_num_logits_per_req
num_packed_cols = max_num_logits_per_req + 2 + (1 if gather_num_nans else 0)
block_size = triton.next_power_of_2(max_num_logits_per_req)
# Pack the sampler output tensors (excluding logprobs) into a single
# tensor for the all-gather.
packed = torch.empty(
metadata.max_num_reqs_per_rank,
num_packed_cols,
dtype=torch.int64,
device=device,
)
if local_output is not None:
assert local_output.num_sampled is not None
assert local_output.num_rejected is not None
assert not gather_num_nans or local_output.num_nans is not None
num_src_cols = min(
local_output.sampled_token_ids.shape[1], max_num_logits_per_req
)
_pack_sampler_output_kernel[(metadata.num_local_reqs,)](
packed,
packed.stride(0),
local_output.sampled_token_ids,
local_output.sampled_token_ids.stride(0),
local_output.num_sampled,
local_output.num_rejected,
local_output.num_nans if gather_num_nans else None,
local_batch.cu_num_logits if gather_num_nans else None,
max_num_logits_per_req,
num_src_cols,
BLOCK_SIZE=block_size,
)
# All-gather the sampler output tensors (excluding logprobs).
gathered = tensor_model_parallel_all_gather(packed, dim=0)
# All-gather the logprobs tensors.
logprobs_tensors = None
if logprobs_dims is not None:
logprobs_tensors = _gather_logprobs_tensors(
local_output, metadata, global_batch, local_batch, logprobs_dims, device
)
# Unpack the gathered tensor into a sampler output object.
num_reqs = metadata.gathered_src_indices.shape[0]
sampled_token_ids = torch.empty(
num_reqs, max_num_logits_per_req, dtype=torch.int64, device=device
)
num_sampled = torch.empty(num_reqs, dtype=torch.int32, device=device)
num_rejected = torch.empty(num_reqs, dtype=torch.int32, device=device)
num_nans = (
torch.empty(num_reqs, dtype=torch.int32, device=device)
if gather_num_nans
else None
)
_unpack_gathered_output_kernel[(num_reqs,)](
gathered,
gathered.stride(0),
metadata.gathered_src_indices,
sampled_token_ids,
num_sampled,
num_rejected,
num_nans,
max_num_logits_per_req,
BLOCK_SIZE=block_size,
)
return SamplerOutput(
sampled_token_ids=sampled_token_ids,
logprobs_tensors=logprobs_tensors,
num_nans=num_nans,
num_sampled=num_sampled,
num_rejected=num_rejected,
)
+20 -8
View File
@@ -103,6 +103,22 @@ class Sampler:
if self.trace_replay_state is not None:
self.trace_replay_state.apply_staged_writes()
def get_logprobs_dims(
self, idx_mapping_np: np.ndarray, include_token_ids: bool = True
) -> tuple[int, int] | None:
"""(num_logprobs, max_per_req_token_ids) for the given requests, or
None when none of them want logprobs."""
max_num_logprobs = self.sampling_states.max_num_logprobs(idx_mapping_np)
max_token_ids = (
self.logprob_token_ids_state.max_num_token_ids(idx_mapping_np)
if include_token_ids
else 0
)
if max_num_logprobs == NO_LOGPROBS and max_token_ids == 0:
return None
num_logprobs = max_num_logprobs if max_num_logprobs != NO_LOGPROBS else 0
return num_logprobs, max_token_ids
def __call__(
self,
logits: torch.Tensor,
@@ -120,11 +136,7 @@ class Sampler:
# that num_nans is computed before applying penalties and temperature.
num_nans = get_num_nans(logits) if self.compute_nans else None
max_num_logprobs = self.sampling_states.max_num_logprobs(idx_mapping_np)
max_per_req_token_ids = self.logprob_token_ids_state.max_num_token_ids(
idx_mapping_np
)
return_logprobs = max_num_logprobs != NO_LOGPROBS or max_per_req_token_ids > 0
logprobs_dims = self.get_logprobs_dims(idx_mapping_np)
sampled, processed_logits = self.sample(
logits,
@@ -134,7 +146,7 @@ class Sampler:
pos,
input_ids,
expanded_local_pos,
return_logprobs=return_logprobs,
return_logprobs=logprobs_dims is not None,
)
if self.trace_replay_state is not None:
@@ -142,12 +154,12 @@ class Sampler:
# computed logprobs reflect the real distribution of the forced token.
self.trace_replay_state.apply_trace(sampled, idx_mapping)
if return_logprobs:
if logprobs_dims is not None:
num_logprobs, max_per_req_token_ids = logprobs_dims
if self.logprobs_mode in PROCESSED_LOGPROBS_MODES:
logits = processed_logits
expanded_logits = logits.shape[0] != idx_mapping_np.shape[0]
cu_num_logits = cu_num_logits_np.tolist() if expanded_logits else None
num_logprobs = max_num_logprobs if max_num_logprobs != NO_LOGPROBS else 0
logprobs_tensors = compute_topk_scores(
logits,
num_logprobs,