[Model Runner V2] batch-sharded sample (#50465)
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
@@ -63,7 +63,7 @@ steps:
|
||||
|
||||
- label: ":nvidia: (L4) Model Runner V2 Distributed"
|
||||
key: model-runner-v2-distributed-2-gpus
|
||||
timeout_in_minutes: 30
|
||||
timeout_in_minutes: 40
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
num_devices: 2
|
||||
source_file_dependencies:
|
||||
@@ -72,6 +72,8 @@ steps:
|
||||
- tests/basic_correctness/test_basic_correctness.py
|
||||
- tests/v1/distributed/test_async_llm_dp.py
|
||||
- tests/v1/distributed/test_eagle_dp.py
|
||||
- tests/v1/e2e/general/test_sharded_sampling.py
|
||||
- tests/v1/e2e/spec_decode/test_sharded_sampling.py
|
||||
commands:
|
||||
- set -x
|
||||
- export VLLM_USE_V2_MODEL_RUNNER=1
|
||||
@@ -81,6 +83,9 @@ steps:
|
||||
- export NCCL_CUMEM_HOST_ENABLE=0
|
||||
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py -k "not ray"
|
||||
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
|
||||
- pytest -v -s v1/e2e/general/test_sharded_sampling.py
|
||||
- pytest -v -s v1/e2e/spec_decode/test_sharded_sampling.py
|
||||
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI300) Model Runner V2 Distributed"
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Batch-sharded sampling must be a drop-in for replicated sampling.
|
||||
|
||||
The comparison is tolerance-based, not bitwise: engine boots land in
|
||||
slightly different kernel/collective states (measured: same-state boots are
|
||||
bitwise identical across the two modes; cross-state boots differ by up to
|
||||
~0.25 logprob and can flip a near-tie token). A real sharding bug — rows
|
||||
routed to the wrong request, misaligned gathers — produces many-nats
|
||||
divergence and scrambled top-k sets, far outside these bounds."""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.utils import multi_gpu_test
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
|
||||
MODEL = "meta-llama/Llama-3.2-1B-Instruct"
|
||||
|
||||
# Boot-state numeric noise bound (measured <= 0.25 on identical contexts).
|
||||
LOGPROB_TOL = 0.5
|
||||
# Near-tie greedy/seeded flips from boot-state noise; measured <= 1 per
|
||||
# engine boot across ~16 completions, plus one of headroom.
|
||||
MAX_DIVERGENT_PROMPTS = 3
|
||||
|
||||
PROMPTS = [
|
||||
"The capital of France is",
|
||||
"The three primary colors are",
|
||||
"Write a haiku about the ocean.",
|
||||
"1, 1, 2, 3, 5, 8,",
|
||||
"The chemical symbol for gold is",
|
||||
"Once upon a time, in a distant kingdom,",
|
||||
"Python's GIL stands for",
|
||||
"The fastest land animal is",
|
||||
]
|
||||
|
||||
# Exercise greedy, seeded random, top-k/top-p, penalties, and logprobs.
|
||||
SAMPLING_PARAMS = [
|
||||
SamplingParams(temperature=0.0, max_tokens=32, logprobs=5, prompt_logprobs=1),
|
||||
SamplingParams(temperature=0.8, seed=42, top_p=0.9, max_tokens=32, logprobs=3),
|
||||
SamplingParams(temperature=0.7, seed=7, top_k=50, min_p=0.05, max_tokens=32),
|
||||
SamplingParams(
|
||||
temperature=0.9,
|
||||
seed=123,
|
||||
frequency_penalty=0.5,
|
||||
repetition_penalty=1.2,
|
||||
max_tokens=32,
|
||||
logprobs=2,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _is_sharded_sampling_active(worker) -> bool:
|
||||
return worker.model_runner.batch_sharder is not None
|
||||
|
||||
|
||||
def _generate(monkeypatch: pytest.MonkeyPatch, disable_sharding: bool):
|
||||
llm = LLM(
|
||||
model=MODEL,
|
||||
tensor_parallel_size=2,
|
||||
enable_batch_sharded_sampling=not disable_sharding,
|
||||
max_model_len=1024,
|
||||
# More requests than ranks, so round-robin slot ownership splits the
|
||||
# batch across both ranks.
|
||||
max_num_seqs=len(PROMPTS),
|
||||
enforce_eager=True,
|
||||
enable_prefix_caching=False,
|
||||
gpu_memory_utilization=0.7,
|
||||
# Timing-based kernel selection is the dominant source of cross-boot
|
||||
# numeric noise; disable it so the comparison bounds stay tight.
|
||||
kernel_config={"enable_flashinfer_autotune": False},
|
||||
)
|
||||
try:
|
||||
# Guard against a vacuous comparison: verify every worker is in the
|
||||
# intended sampling mode.
|
||||
modes = llm.llm_engine.collective_rpc(_is_sharded_sampling_active)
|
||||
assert all(mode == (not disable_sharding) for mode in modes), modes
|
||||
params = [
|
||||
SAMPLING_PARAMS[i % len(SAMPLING_PARAMS)] for i in range(len(PROMPTS))
|
||||
]
|
||||
# Two waves: the second reuses the request slots freed by the first
|
||||
# (in whatever order requests finished), covering the cross-rank
|
||||
# slot-recycling path that request ownership derives from.
|
||||
wave1 = llm.generate(PROMPTS, params)
|
||||
wave2 = llm.generate(PROMPTS[::-1], params[::-1])
|
||||
return wave1 + wave2
|
||||
finally:
|
||||
del llm
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
def _assert_logprob_dicts_close(ref_lps, out_lps, what: str) -> None:
|
||||
# Allow one boundary entry of the top-k set to swap at a near-tie.
|
||||
common = set(ref_lps) & set(out_lps)
|
||||
assert len(common) >= max(len(ref_lps), len(out_lps)) - 1, (
|
||||
f"{what}: top-k token sets diverge: {sorted(ref_lps)} vs {sorted(out_lps)}"
|
||||
)
|
||||
for token_id in common:
|
||||
diff = abs(ref_lps[token_id].logprob - out_lps[token_id].logprob)
|
||||
assert diff <= LOGPROB_TOL, (
|
||||
f"{what}[{token_id}]: logprob diff {diff} "
|
||||
f"({ref_lps[token_id].logprob} vs {out_lps[token_id].logprob})"
|
||||
)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_sharded_sampling_outputs_match(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Generation, logprobs, and prompt logprobs match between batch-sharded
|
||||
sampling and the replicated fallback, up to measured boot-state noise."""
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
# Required for the collective_rpc mode check below.
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
|
||||
ref_outputs = _generate(monkeypatch, disable_sharding=True)
|
||||
shard_outputs = _generate(monkeypatch, disable_sharding=False)
|
||||
|
||||
assert len(ref_outputs) == len(shard_outputs) == 2 * len(PROMPTS)
|
||||
num_divergent = 0
|
||||
for i, (ref, out) in enumerate(zip(ref_outputs, shard_outputs)):
|
||||
ref_completion = ref.outputs[0]
|
||||
out_completion = out.outputs[0]
|
||||
ref_ids = list(ref_completion.token_ids)
|
||||
out_ids = list(out_completion.token_ids)
|
||||
|
||||
# Logits are only comparable at positions whose context (prompt +
|
||||
# previously sampled tokens) is identical: every position up to and
|
||||
# including the first divergence.
|
||||
prefix = 0
|
||||
while prefix < min(len(ref_ids), len(out_ids)):
|
||||
if ref_ids[prefix] != out_ids[prefix]:
|
||||
break
|
||||
prefix += 1
|
||||
if ref_ids != out_ids:
|
||||
num_divergent += 1
|
||||
|
||||
if ref_completion.logprobs is not None:
|
||||
assert out_completion.logprobs is not None
|
||||
comparable = min(prefix + 1, len(ref_ids), len(out_ids))
|
||||
for pos in range(comparable):
|
||||
_assert_logprob_dicts_close(
|
||||
ref_completion.logprobs[pos],
|
||||
out_completion.logprobs[pos],
|
||||
f"prompt {i} logprobs[{pos}]",
|
||||
)
|
||||
|
||||
# The prompt is fixed, so every prompt-logprob position is comparable.
|
||||
if ref.prompt_logprobs is not None:
|
||||
assert out.prompt_logprobs is not None
|
||||
for pos, (ref_lps, out_lps) in enumerate(
|
||||
zip(ref.prompt_logprobs, out.prompt_logprobs)
|
||||
):
|
||||
if ref_lps is None or out_lps is None:
|
||||
assert ref_lps is None and out_lps is None
|
||||
continue
|
||||
_assert_logprob_dicts_close(
|
||||
ref_lps, out_lps, f"prompt {i} prompt_logprobs[{pos}]"
|
||||
)
|
||||
|
||||
assert num_divergent <= MAX_DIVERGENT_PROMPTS, (
|
||||
f"{num_divergent}/{2 * len(PROMPTS)} prompts diverged: beyond near-tie "
|
||||
"boot-state noise; sharded sampling is likely misrouting requests"
|
||||
)
|
||||
@@ -0,0 +1,102 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Batch-sharded sampling must be a drop-in for replicated sampling under
|
||||
speculative decoding.
|
||||
|
||||
The non-spec arm of this claim lives in tests/v1/e2e/general/
|
||||
test_sharded_sampling.py. Spec decode is the harder case: draft tokens give
|
||||
each request a different number of logits rows, so the shard plan's
|
||||
per-request arithmetic and the rejection sampler are both exercised.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.utils import wait_for_memory_to_settle
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
|
||||
from .utils import _skip_if_insufficient_gpus_for_tp, get_test_prompts, greedy_sampling
|
||||
|
||||
|
||||
def _is_sharded_sampling_active(worker) -> bool:
|
||||
return worker.model_runner.batch_sharder is not None
|
||||
|
||||
|
||||
def test_mtp_sharded_sampling_equivalence(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Batch-sharded sampling must be a bit-exact drop-in for replicated
|
||||
sampling under MTP spec decoding: the collectives move the same logits
|
||||
bytes, Gumbel keys derive from (request slot, position, seed), and slot
|
||||
assignment is rank-deterministic. Both runs here are spec decode with
|
||||
identical math, so outputs must match exactly."""
|
||||
tp_size = 2
|
||||
_skip_if_insufficient_gpus_for_tp(tp_size)
|
||||
model_name = "Qwen/Qwen3.5-0.8B-Base"
|
||||
test_prompts = get_test_prompts(mm_enabled=False, num_prompts=20)
|
||||
|
||||
with monkeypatch.context() as m:
|
||||
# Batch-sharded sampling only exists in the V2 model runner.
|
||||
m.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
# Required for the collective_rpc mode check below.
|
||||
m.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
|
||||
def run(disable_sharding: bool):
|
||||
llm = LLM(
|
||||
model=model_name,
|
||||
tensor_parallel_size=tp_size,
|
||||
enable_batch_sharded_sampling=not disable_sharding,
|
||||
speculative_config={
|
||||
"method": "mtp",
|
||||
"num_speculative_tokens": 1,
|
||||
"max_model_len": 2048,
|
||||
},
|
||||
max_model_len=2048,
|
||||
limit_mm_per_prompt={"image": 0, "video": 0},
|
||||
# More requests than ranks, so round-robin slot ownership
|
||||
# splits the batch across both ranks.
|
||||
max_num_seqs=len(test_prompts),
|
||||
# Timing-based kernel selection is the dominant source of
|
||||
# cross-boot numeric noise; disable for a tight comparison.
|
||||
kernel_config={"enable_flashinfer_autotune": False},
|
||||
)
|
||||
try:
|
||||
# Guard against a vacuous comparison: verify every worker is
|
||||
# in the intended sampling mode.
|
||||
modes = llm.llm_engine.collective_rpc(_is_sharded_sampling_active)
|
||||
assert all(mode == (not disable_sharding) for mode in modes)
|
||||
greedy = llm.chat(test_prompts, greedy_sampling())
|
||||
seeded = llm.chat(
|
||||
test_prompts,
|
||||
SamplingParams(temperature=1.0, seed=33, max_tokens=10),
|
||||
)
|
||||
return greedy, seeded
|
||||
finally:
|
||||
del llm
|
||||
torch.accelerator.empty_cache()
|
||||
cleanup_dist_env_and_memory()
|
||||
wait_for_memory_to_settle()
|
||||
|
||||
ref_greedy, ref_seeded = run(disable_sharding=True)
|
||||
shard_greedy, shard_seeded = run(disable_sharding=False)
|
||||
|
||||
# Engine boots are not bitwise deterministic (kernel/collective
|
||||
# selection shifts logits by ~1 ulp, flipping near-tie tokens), so
|
||||
# allow a small number of divergent prompts. A sharding bug produces
|
||||
# wholesale divergence, not isolated near-tie flips.
|
||||
for name, ref_outputs, shard_outputs in (
|
||||
("greedy", ref_greedy, shard_greedy),
|
||||
("seeded", ref_seeded, shard_seeded),
|
||||
):
|
||||
num_divergent = 0
|
||||
for i, (ref, out) in enumerate(zip(ref_outputs, shard_outputs)):
|
||||
if list(ref.outputs[0].token_ids) != list(out.outputs[0].token_ids):
|
||||
num_divergent += 1
|
||||
print(
|
||||
f"{name} prompt {i} diverged:\n"
|
||||
f" replicated: {ref.outputs[0].text!r}\n"
|
||||
f" sharded: {out.outputs[0].text!r}"
|
||||
)
|
||||
assert num_divergent <= 2, (
|
||||
f"{name}: {num_divergent}/{len(ref_outputs)} prompts diverged, "
|
||||
"beyond near-tie boot noise"
|
||||
)
|
||||
@@ -0,0 +1,613 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Routing-math tests for batch-sharded sampling.
|
||||
|
||||
These simulate every TP rank in-process (no torch.distributed): the local
|
||||
sub-batch, the all-to-all send/recv layout, and the padded result gather must
|
||||
compose to a bit-exact round trip of the replicated computation.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.v1.core.sched.output import GrammarOutput
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.sample import batch_shard
|
||||
from vllm.v1.worker.gpu.sample.batch_shard import (
|
||||
BatchSharder,
|
||||
_shard_grammar_output,
|
||||
)
|
||||
|
||||
DEVICE = "cuda"
|
||||
|
||||
# The owner-sorted layout is expanded by a Triton kernel, so the routing math
|
||||
# can only be exercised on device.
|
||||
requires_cuda = pytest.mark.skipif(
|
||||
not torch.cuda.is_available(), reason="requires CUDA"
|
||||
)
|
||||
|
||||
|
||||
def _np(t: torch.Tensor) -> np.ndarray:
|
||||
return t.cpu().numpy()
|
||||
|
||||
|
||||
def _make_batch(
|
||||
rng: np.random.Generator,
|
||||
num_reqs: int,
|
||||
max_num_reqs: int,
|
||||
max_spec: int,
|
||||
slot_pool: np.ndarray | None = None,
|
||||
) -> InputBatch:
|
||||
"""Consistent batch with random unique request slots, 1 + k_i logits
|
||||
rows per request at the tail of each query segment."""
|
||||
if slot_pool is None:
|
||||
slot_pool = np.arange(max_num_reqs)
|
||||
slots = rng.choice(slot_pool, size=num_reqs, replace=False)
|
||||
slots = slots.astype(np.int32)
|
||||
k = rng.integers(0, max_spec + 1, size=num_reqs)
|
||||
num_logits_per_req = (1 + k).astype(np.int32)
|
||||
cu = np.zeros(num_reqs + 1, dtype=np.int32)
|
||||
np.cumsum(num_logits_per_req, out=cu[1:])
|
||||
|
||||
query_lens = num_logits_per_req + rng.integers(0, 4, size=num_reqs).astype(np.int32)
|
||||
qsl = np.zeros(num_reqs + 1, dtype=np.int32)
|
||||
np.cumsum(query_lens, out=qsl[1:])
|
||||
num_tokens = int(qsl[-1])
|
||||
|
||||
logits_indices_np = np.concatenate(
|
||||
[
|
||||
np.arange(qsl[i + 1] - num_logits_per_req[i], qsl[i + 1])
|
||||
for i in range(num_reqs)
|
||||
]
|
||||
).astype(np.int64)
|
||||
expanded_idx_np = np.repeat(slots, num_logits_per_req)
|
||||
expanded_pos_np = np.concatenate(
|
||||
[np.arange(n, dtype=np.int32) for n in num_logits_per_req]
|
||||
)
|
||||
seq_lens_np = (query_lens + rng.integers(0, 100, size=num_reqs)).astype(np.int32)
|
||||
|
||||
def dev(a: np.ndarray) -> torch.Tensor:
|
||||
return torch.from_numpy(a).to(DEVICE)
|
||||
|
||||
return InputBatch(
|
||||
req_ids=[f"req-{i}" for i in range(num_reqs)],
|
||||
num_reqs=num_reqs,
|
||||
num_reqs_after_padding=num_reqs,
|
||||
idx_mapping=dev(slots),
|
||||
idx_mapping_np=slots,
|
||||
expanded_idx_mapping=dev(expanded_idx_np),
|
||||
expanded_local_pos=dev(expanded_pos_np),
|
||||
num_scheduled_tokens=query_lens,
|
||||
num_tokens=num_tokens,
|
||||
num_tokens_after_padding=num_tokens,
|
||||
num_draft_tokens=int(k.sum()),
|
||||
num_draft_tokens_per_req=k.astype(np.int32),
|
||||
query_start_loc=dev(qsl),
|
||||
query_start_loc_np=qsl,
|
||||
seq_lens=dev(seq_lens_np),
|
||||
seq_lens_cpu_upper_bound=torch.from_numpy(seq_lens_np.copy()),
|
||||
dcp_local_seq_lens=None,
|
||||
num_computed_tokens_np=np.zeros(num_reqs, dtype=np.int32),
|
||||
prefill_len_np=np.zeros(num_reqs, dtype=np.int32),
|
||||
num_computed_prefill_tokens_np=np.zeros(num_reqs, dtype=np.int32),
|
||||
is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_),
|
||||
has_prefill=False,
|
||||
max_seq_len_np=None,
|
||||
input_ids=torch.zeros(num_tokens, dtype=torch.int32, device=DEVICE),
|
||||
positions=torch.arange(num_tokens, dtype=torch.int64, device=DEVICE),
|
||||
is_padding=torch.zeros(num_tokens, dtype=torch.bool, device=DEVICE),
|
||||
logits_indices=dev(logits_indices_np),
|
||||
cu_num_logits=dev(cu),
|
||||
cu_num_logits_np=cu,
|
||||
has_structured_output_reqs=False,
|
||||
prompt_lens=None,
|
||||
)
|
||||
|
||||
|
||||
def _shard_all_ranks(batch: InputBatch, max_num_reqs: int, tp_size: int):
|
||||
"""Simulate every TP rank by faking get_tp_group() per BatchSharder."""
|
||||
results = []
|
||||
for rank in range(tp_size):
|
||||
fake_group = SimpleNamespace(rank_in_group=rank, world_size=tp_size)
|
||||
with mock.patch.object(batch_shard, "get_tp_group", return_value=fake_group):
|
||||
sharder = BatchSharder(
|
||||
max_num_reqs=max_num_reqs,
|
||||
max_num_logits_per_req=8,
|
||||
device=torch.device(DEVICE),
|
||||
)
|
||||
local_batch, sorted_logits_indices, _, metadata = sharder.shard_sampler_inputs(
|
||||
batch, None
|
||||
)
|
||||
results.append((local_batch, sorted_logits_indices, metadata))
|
||||
return results
|
||||
|
||||
|
||||
def _owned_batch_indices(local_batch: InputBatch) -> np.ndarray:
|
||||
return np.array([int(r.split("-")[1]) for r in local_batch.req_ids], dtype=np.int64)
|
||||
|
||||
|
||||
def _owned_rows(cu: np.ndarray, owned: np.ndarray) -> np.ndarray:
|
||||
if len(owned) == 0:
|
||||
return np.array([], dtype=np.int64)
|
||||
return np.concatenate([np.arange(cu[r], cu[r + 1]) for r in owned])
|
||||
|
||||
|
||||
@requires_cuda
|
||||
@pytest.mark.parametrize("tp_size", [2, 4, 8])
|
||||
@pytest.mark.parametrize("seed", [0, 1])
|
||||
def test_local_batch_partition(tp_size: int, seed: int):
|
||||
"""Local sub-batches partition the batch by owner, preserve batch order
|
||||
within an owner, and carry exactly the owned requests' entries of every
|
||||
per-request and per-row array."""
|
||||
rng = np.random.default_rng(seed)
|
||||
max_num_reqs = 64
|
||||
batch = _make_batch(rng, num_reqs=23, max_num_reqs=max_num_reqs, max_spec=3)
|
||||
num_logits = int(batch.cu_num_logits_np[-1])
|
||||
results = _shard_all_ranks(batch, max_num_reqs, tp_size)
|
||||
cu = batch.cu_num_logits_np
|
||||
|
||||
metadata0 = results[0][2]
|
||||
assert sum(metadata0.num_logits_per_rank) == num_logits
|
||||
# Gather width = max logits rows of any request (bounds num_sampled).
|
||||
assert metadata0.max_num_logits_per_req == int(np.diff(cu).max())
|
||||
|
||||
all_owned: list[int] = []
|
||||
for rank, (local, _, metadata) in enumerate(results):
|
||||
owned = _owned_batch_indices(local)
|
||||
assert local.num_reqs == len(owned) == metadata.num_local_reqs
|
||||
assert (batch.idx_mapping_np[owned] % tp_size == rank).all()
|
||||
# Stable sort: batch order is preserved within an owner.
|
||||
assert (np.diff(owned) > 0).all()
|
||||
all_owned.extend(owned.tolist())
|
||||
|
||||
rows = _owned_rows(cu, owned)
|
||||
assert (
|
||||
metadata.num_local_logits
|
||||
== len(rows)
|
||||
== metadata0.num_logits_per_rank[rank]
|
||||
)
|
||||
|
||||
expected_idx = batch.idx_mapping_np[owned]
|
||||
assert (local.idx_mapping_np == expected_idx).all()
|
||||
assert _np(local.idx_mapping).tolist() == expected_idx.tolist()
|
||||
assert (np.diff(local.cu_num_logits_np) == np.diff(cu)[owned]).all()
|
||||
assert _np(local.cu_num_logits).tolist() == local.cu_num_logits_np.tolist()
|
||||
assert (
|
||||
_np(local.expanded_idx_mapping) == _np(batch.expanded_idx_mapping)[rows]
|
||||
).all()
|
||||
assert (
|
||||
_np(local.expanded_local_pos) == _np(batch.expanded_local_pos)[rows]
|
||||
).all()
|
||||
assert (_np(local.logits_indices) == _np(batch.logits_indices)[rows]).all()
|
||||
assert (_np(local.seq_lens) == _np(batch.seq_lens)[owned]).all()
|
||||
assert batch.num_draft_tokens_per_req is not None
|
||||
assert local.num_draft_tokens_per_req is not None
|
||||
assert (
|
||||
local.num_draft_tokens_per_req == batch.num_draft_tokens_per_req[owned]
|
||||
).all()
|
||||
assert local.num_draft_tokens == int(
|
||||
batch.num_draft_tokens_per_req[owned].sum()
|
||||
)
|
||||
# Triton specializes kernels on pointer alignment: every packed-buffer
|
||||
# view fed to kernels must stay 16-byte aligned across batch shapes,
|
||||
# or serving JITs new kernel variants mid-collective.
|
||||
for t in (
|
||||
local.idx_mapping,
|
||||
local.expanded_idx_mapping,
|
||||
local.expanded_local_pos,
|
||||
local.logits_indices,
|
||||
local.cu_num_logits,
|
||||
metadata.gathered_src_indices,
|
||||
):
|
||||
assert t.data_ptr() % 16 == 0
|
||||
assert sorted(all_owned) == list(range(batch.num_reqs))
|
||||
|
||||
# sorted_logits_indices is identical on all ranks: the owner-sorted logits row
|
||||
# positions, i.e. logits_indices permuted rank-major.
|
||||
sorted_logits_indices = _np(results[0][1])
|
||||
for _, gi, _ in results:
|
||||
assert (_np(gi) == sorted_logits_indices).all()
|
||||
rows_all = np.concatenate(
|
||||
[_owned_rows(cu, _owned_batch_indices(local)) for local, _, _ in results]
|
||||
)
|
||||
assert (
|
||||
sorted_logits_indices.tolist() == _np(batch.logits_indices)[rows_all].tolist()
|
||||
)
|
||||
|
||||
|
||||
@requires_cuda
|
||||
@pytest.mark.parametrize("tp_size", [2, 4])
|
||||
def test_all_to_all_reassembly_simulated(tp_size: int):
|
||||
"""Vocab shards computed from owner-sorted hidden states and routed
|
||||
through the simulated all-to-all reassemble into exactly the full-vocab
|
||||
logits of each rank's owned rows."""
|
||||
rng = np.random.default_rng(7)
|
||||
max_num_reqs = 32
|
||||
batch = _make_batch(rng, num_reqs=11, max_num_reqs=max_num_reqs, max_spec=4)
|
||||
vocab = 64 * tp_size
|
||||
shard_w = vocab // tp_size
|
||||
token_logits = torch.randn(batch.num_tokens, vocab, device=DEVICE)
|
||||
|
||||
results = _shard_all_ranks(batch, max_num_reqs, tp_size)
|
||||
sorted_logits_indices = results[0][1]
|
||||
num_logits_per_rank = results[0][2].num_logits_per_rank
|
||||
bounds = np.zeros(tp_size + 1, dtype=np.int64)
|
||||
np.cumsum(num_logits_per_rank, out=bounds[1:])
|
||||
|
||||
# send_r = rank r's vocab shard of the owner-sorted rows, with no further
|
||||
# permute; chunk d of send_r goes to rank d.
|
||||
sends = [
|
||||
token_logits[sorted_logits_indices][:, r * shard_w : (r + 1) * shard_w]
|
||||
for r in range(tp_size)
|
||||
]
|
||||
for rank, (local, _, metadata) in enumerate(results):
|
||||
recv = torch.cat(
|
||||
[sends[src][bounds[rank] : bounds[rank + 1]] for src in range(tp_size)]
|
||||
)
|
||||
# Reassemble source-rank-major chunks into full-vocab rows, mirroring
|
||||
# all_to_all_logits (source-rank order is vocab-shard order).
|
||||
reassembled = (
|
||||
recv.view(tp_size, metadata.num_local_logits, shard_w)
|
||||
.permute(1, 0, 2)
|
||||
.reshape(metadata.num_local_logits, vocab)
|
||||
)
|
||||
expected = token_logits[local.logits_indices]
|
||||
assert torch.equal(reassembled, expected)
|
||||
|
||||
|
||||
@requires_cuda
|
||||
def test_shard_no_spec():
|
||||
"""Batches without draft tokens (one logits row per request) degenerate
|
||||
cleanly: rows mirror requests."""
|
||||
rng = np.random.default_rng(11)
|
||||
tp_size = 4
|
||||
max_num_reqs = 32
|
||||
batch = _make_batch(rng, num_reqs=10, max_num_reqs=max_num_reqs, max_spec=0)
|
||||
results = _shard_all_ranks(batch, max_num_reqs, tp_size)
|
||||
|
||||
assert results[0][2].max_num_logits_per_req == 1
|
||||
for local, _, metadata in results:
|
||||
assert metadata.num_local_logits == metadata.num_local_reqs == local.num_reqs
|
||||
owned = _owned_batch_indices(local)
|
||||
assert _np(local.logits_indices).tolist() == (
|
||||
_np(batch.logits_indices)[owned].tolist()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [2, 4])
|
||||
def test_shard_grammar_output(tp_size: int):
|
||||
"""The grammar filter keeps exactly the owned requests' bitmask rows, in
|
||||
grammar order, and the per-rank pieces partition the global bitmask."""
|
||||
rng = np.random.default_rng(5)
|
||||
max_num_reqs = 32
|
||||
num_reqs = 12
|
||||
slots = rng.choice(np.arange(max_num_reqs), size=num_reqs, replace=False)
|
||||
k = rng.integers(0, 4, size=num_reqs)
|
||||
cu = np.zeros(num_reqs + 1, dtype=np.int32)
|
||||
np.cumsum(1 + k, out=cu[1:])
|
||||
owner = slots % tp_size
|
||||
|
||||
# Grammar requests: every other request, in a scrambled order.
|
||||
req_ids = [f"req-{i}" for i in range(num_reqs)]
|
||||
grammar_idx = list(rng.permutation(np.arange(0, num_reqs, 2)))
|
||||
grammar_req_ids = [req_ids[i] for i in grammar_idx]
|
||||
num_grammar_rows = sum(int(cu[i + 1] - cu[i]) for i in grammar_idx)
|
||||
bitmask = rng.integers(0, 2**31, size=(num_grammar_rows, 4), dtype=np.int32)
|
||||
input_batch = SimpleNamespace(req_ids=req_ids, cu_num_logits_np=cu)
|
||||
|
||||
# Global row offset of each grammar request within the bitmask.
|
||||
row_offsets = {}
|
||||
cursor = 0
|
||||
for i in grammar_idx:
|
||||
row_offsets[i] = cursor
|
||||
cursor += int(cu[i + 1] - cu[i])
|
||||
|
||||
grammar_output = GrammarOutput(
|
||||
structured_output_request_ids=grammar_req_ids,
|
||||
grammar_bitmask=bitmask,
|
||||
)
|
||||
|
||||
kept_total = 0
|
||||
for rank in range(tp_size):
|
||||
local_req_ids = [req_ids[i] for i in range(num_reqs) if owner[i] == rank]
|
||||
local = _shard_grammar_output(grammar_output, input_batch, local_req_ids)
|
||||
expected_idx = [i for i in grammar_idx if owner[i] == rank]
|
||||
if not expected_idx:
|
||||
assert local is None
|
||||
continue
|
||||
assert local is not None
|
||||
assert local.structured_output_request_ids == [req_ids[i] for i in expected_idx]
|
||||
expected_rows = np.concatenate(
|
||||
[
|
||||
np.arange(row_offsets[i], row_offsets[i] + cu[i + 1] - cu[i])
|
||||
for i in expected_idx
|
||||
]
|
||||
)
|
||||
assert (local.grammar_bitmask == bitmask[expected_rows.astype(int)]).all()
|
||||
kept_total += local.grammar_bitmask.shape[0]
|
||||
assert kept_total == num_grammar_rows
|
||||
|
||||
|
||||
@requires_cuda
|
||||
@pytest.mark.parametrize("tp_size", [2, 4])
|
||||
def test_gather_src_indices_round_trip(tp_size: int):
|
||||
"""Padded per-rank result blocks indexed by gathered_src_indices restore
|
||||
batch order, including when some rank owns zero requests."""
|
||||
rng = np.random.default_rng(3)
|
||||
max_num_reqs = 40
|
||||
all_slots = np.arange(max_num_reqs)
|
||||
batch = _make_batch(
|
||||
rng,
|
||||
num_reqs=17,
|
||||
max_num_reqs=max_num_reqs,
|
||||
max_spec=2,
|
||||
slot_pool=all_slots[all_slots % tp_size != 0], # rank 0 owns nothing
|
||||
)
|
||||
results = _shard_all_ranks(batch, max_num_reqs, tp_size)
|
||||
metadata0 = results[0][2]
|
||||
assert metadata0.num_local_reqs == 0
|
||||
|
||||
# Value of request j is j; each rank packs its owned requests' values into
|
||||
# a zero-padded (max_num_reqs_per_rank, 1) block, gathered rank-major.
|
||||
pad = metadata0.max_num_reqs_per_rank
|
||||
blocks = []
|
||||
for local, _, _ in results:
|
||||
owned = _owned_batch_indices(local)
|
||||
block = torch.zeros(pad, 1, dtype=torch.int64, device=DEVICE)
|
||||
block[: len(owned), 0] = torch.from_numpy(owned).to(DEVICE)
|
||||
blocks.append(block)
|
||||
gathered = torch.cat(blocks)
|
||||
|
||||
restored = gathered[metadata0.gathered_src_indices][:, 0]
|
||||
assert restored.tolist() == list(range(batch.num_reqs))
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
@pytest.mark.parametrize("tp_size", [2, 4])
|
||||
def test_gather_sampler_output_kernels(tp_size: int):
|
||||
"""The fused pack/unpack kernels round-trip per-rank sampler results
|
||||
into batch order: token ids zero-filled past the local width, counts
|
||||
cast back to int32, identical on every rank."""
|
||||
from dataclasses import replace as dc_replace
|
||||
|
||||
from vllm.v1.worker.gpu.sample.output import SamplerOutput
|
||||
|
||||
rng = np.random.default_rng(9)
|
||||
max_num_reqs = 32
|
||||
batch = _make_batch(rng, num_reqs=13, max_num_reqs=max_num_reqs, max_spec=3)
|
||||
results = _shard_all_ranks(batch, max_num_reqs, tp_size)
|
||||
device = torch.device("cuda")
|
||||
num_reqs = batch.num_reqs
|
||||
width = results[0][2].max_num_logits_per_req
|
||||
|
||||
# Distinctive per-rank local outputs. Even ranks mimic the regular
|
||||
# sampler (one column, exercising zero-fill); odd ranks mimic the
|
||||
# rejection sampler's over-allocation (exercising truncation).
|
||||
local_outputs: list[SamplerOutput | None] = []
|
||||
expected_ids = torch.zeros(num_reqs, width, dtype=torch.int64)
|
||||
expected_sampled = torch.zeros(num_reqs, dtype=torch.int32)
|
||||
expected_rejected = torch.zeros(num_reqs, dtype=torch.int32)
|
||||
for rank, (local, _, metadata) in enumerate(results):
|
||||
if metadata.num_local_reqs == 0:
|
||||
local_outputs.append(None)
|
||||
continue
|
||||
owned = torch.from_numpy(_owned_batch_indices(local))
|
||||
src_width = 1 if rank % 2 == 0 else width + 3
|
||||
ids = owned[:, None] * 1000 + torch.arange(src_width)[None, :]
|
||||
num_sampled = (owned % width + 1).to(torch.int32)
|
||||
num_rejected = (owned % 3).to(torch.int32)
|
||||
local_outputs.append(
|
||||
SamplerOutput(
|
||||
sampled_token_ids=ids.to(device),
|
||||
logprobs_tensors=None,
|
||||
num_nans=None,
|
||||
num_sampled=num_sampled.to(device),
|
||||
num_rejected=num_rejected.to(device),
|
||||
)
|
||||
)
|
||||
copy_width = min(src_width, width)
|
||||
expected_ids[owned, :copy_width] = ids[:, :copy_width]
|
||||
expected_sampled[owned] = num_sampled
|
||||
expected_rejected[owned] = num_rejected
|
||||
|
||||
# Two passes over a mocked all-gather: capture each rank's packed send
|
||||
# block, then replay with the concatenated blocks as the gathered result.
|
||||
packed_blocks: dict[int, torch.Tensor] = {}
|
||||
gathered_full: list[torch.Tensor] = []
|
||||
|
||||
def run(rank: int) -> SamplerOutput:
|
||||
metadata = dc_replace(
|
||||
results[rank][2],
|
||||
gathered_src_indices=results[rank][2].gathered_src_indices.to(device),
|
||||
)
|
||||
|
||||
def fake_all_gather(x: torch.Tensor, dim: int = 0) -> torch.Tensor:
|
||||
packed_blocks[rank] = x.clone()
|
||||
if gathered_full:
|
||||
return gathered_full[0]
|
||||
return torch.zeros(
|
||||
tp_size * x.shape[0], x.shape[1], dtype=x.dtype, device=x.device
|
||||
)
|
||||
|
||||
with mock.patch.object(
|
||||
batch_shard, "tensor_model_parallel_all_gather", fake_all_gather
|
||||
):
|
||||
return batch_shard.gather_sampler_output(
|
||||
local_outputs[rank],
|
||||
metadata,
|
||||
device,
|
||||
global_batch=batch,
|
||||
local_batch=results[rank][0],
|
||||
)
|
||||
|
||||
for rank in range(tp_size):
|
||||
run(rank)
|
||||
# Zero the uninitialized padding rows so the comparison is deterministic;
|
||||
# the real path never reads them (gathered_src_indices skips padding).
|
||||
for rank, (local, _, metadata) in enumerate(results):
|
||||
packed_blocks[rank][metadata.num_local_reqs :] = 0
|
||||
gathered_full.append(torch.cat([packed_blocks[r] for r in range(tp_size)]))
|
||||
|
||||
for rank in range(tp_size):
|
||||
out = run(rank)
|
||||
assert out.sampled_token_ids.dtype == torch.int64
|
||||
assert out.num_sampled is not None and out.num_sampled.dtype == torch.int32
|
||||
assert out.num_rejected is not None
|
||||
assert torch.equal(out.sampled_token_ids.cpu(), expected_ids)
|
||||
assert torch.equal(out.num_sampled.cpu(), expected_sampled)
|
||||
assert torch.equal(out.num_rejected.cpu(), expected_rejected.to(torch.int32))
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_gather_sampler_output_logprobs_and_nans():
|
||||
"""num_nans is reduced per request and gathered; LogprobsTensors are
|
||||
packed into fixed per-request blocks, width-aligned across ranks (pad
|
||||
and truncate paths), and restored to global row order."""
|
||||
from dataclasses import replace as dc_replace
|
||||
|
||||
from vllm.v1.outputs import LogprobsTensors
|
||||
from vllm.v1.worker.gpu.sample.output import SamplerOutput
|
||||
|
||||
tp_size = 2
|
||||
rng = np.random.default_rng(21)
|
||||
max_num_reqs = 32
|
||||
batch = _make_batch(rng, num_reqs=6, max_num_reqs=max_num_reqs, max_spec=3)
|
||||
results = _shard_all_ranks(batch, max_num_reqs, tp_size)
|
||||
device = torch.device("cuda")
|
||||
num_reqs = batch.num_reqs
|
||||
cu = batch.cu_num_logits_np
|
||||
num_logits = int(cu[-1])
|
||||
width = results[0][2].max_num_logits_per_req
|
||||
num_cols = 3
|
||||
|
||||
def cuda_fields(b: InputBatch) -> InputBatch:
|
||||
return dc_replace(
|
||||
b,
|
||||
cu_num_logits=b.cu_num_logits.to(device),
|
||||
expanded_local_pos=b.expanded_local_pos.to(device),
|
||||
)
|
||||
|
||||
global_cuda = cuda_fields(batch)
|
||||
|
||||
local_outputs: list[SamplerOutput | None] = []
|
||||
local_batches: list[InputBatch] = []
|
||||
expected_ids = torch.zeros(num_logits, num_cols, dtype=torch.int64)
|
||||
expected_logprobs = torch.full(
|
||||
(num_logits, num_cols), float("-inf"), dtype=torch.float32
|
||||
)
|
||||
expected_ranks = torch.zeros(num_logits, dtype=torch.int64)
|
||||
expected_nans = torch.zeros(num_reqs, dtype=torch.int32)
|
||||
for rank, (local, _, metadata) in enumerate(results):
|
||||
local_batches.append(cuda_fields(local))
|
||||
if metadata.num_local_reqs == 0:
|
||||
local_outputs.append(None)
|
||||
continue
|
||||
owned = _owned_batch_indices(local)
|
||||
rows = torch.from_numpy(_owned_rows(cu, owned))
|
||||
# Rank 0 sends fewer columns (pad path), rank 1 more (truncate path).
|
||||
local_cols = num_cols - 1 if rank == 0 else num_cols + 2
|
||||
ids = rows[:, None] * 10 + torch.arange(local_cols)[None, :]
|
||||
logprobs = ids.float() * 0.5
|
||||
ranks_t = rows + 7
|
||||
nans = (rows % 4).to(torch.int32)
|
||||
local_outputs.append(
|
||||
SamplerOutput(
|
||||
sampled_token_ids=torch.zeros(
|
||||
metadata.num_local_reqs, width, dtype=torch.int64, device=device
|
||||
),
|
||||
logprobs_tensors=LogprobsTensors(
|
||||
logprob_token_ids=ids.to(device),
|
||||
logprobs=logprobs.to(device),
|
||||
selected_token_ranks=ranks_t.to(device),
|
||||
cu_num_generated_tokens=None,
|
||||
),
|
||||
num_nans=nans.to(device),
|
||||
num_sampled=torch.ones(
|
||||
metadata.num_local_reqs, dtype=torch.int32, device=device
|
||||
),
|
||||
num_rejected=torch.zeros(
|
||||
metadata.num_local_reqs, dtype=torch.int32, device=device
|
||||
),
|
||||
)
|
||||
)
|
||||
copy_cols = min(local_cols, num_cols)
|
||||
expected_ids[rows, :copy_cols] = ids[:, :copy_cols]
|
||||
expected_logprobs[rows, :copy_cols] = logprobs[:, :copy_cols]
|
||||
expected_ranks[rows] = ranks_t
|
||||
for j in owned:
|
||||
in_req = (rows >= int(cu[j])) & (rows < int(cu[j + 1]))
|
||||
expected_nans[j] = int(nans[in_req].sum())
|
||||
|
||||
recorded: dict[int, list[torch.Tensor]] = {}
|
||||
gathered_full: list[torch.Tensor] = []
|
||||
|
||||
def run(rank: int) -> SamplerOutput:
|
||||
metadata = dc_replace(
|
||||
results[rank][2],
|
||||
gathered_src_indices=results[rank][2].gathered_src_indices.to(device),
|
||||
)
|
||||
calls = {"i": 0}
|
||||
|
||||
def fake_all_gather(x: torch.Tensor, dim: int = 0) -> torch.Tensor:
|
||||
i = calls["i"]
|
||||
calls["i"] += 1
|
||||
if gathered_full:
|
||||
return gathered_full[i]
|
||||
recorded.setdefault(rank, []).append(x.clone())
|
||||
return torch.zeros(
|
||||
tp_size * x.shape[0], *x.shape[1:], dtype=x.dtype, device=x.device
|
||||
)
|
||||
|
||||
with mock.patch.object(
|
||||
batch_shard, "tensor_model_parallel_all_gather", fake_all_gather
|
||||
):
|
||||
return batch_shard.gather_sampler_output(
|
||||
local_outputs[rank],
|
||||
metadata,
|
||||
device,
|
||||
global_batch=global_cuda,
|
||||
local_batch=local_batches[rank],
|
||||
gather_num_nans=True,
|
||||
logprobs_dims=(num_cols - 1, 0),
|
||||
)
|
||||
|
||||
for rank in range(tp_size):
|
||||
run(rank)
|
||||
gathered_full.extend(
|
||||
torch.cat([recorded[r][i] for r in range(tp_size)]) for i in range(3)
|
||||
)
|
||||
|
||||
for rank in range(tp_size):
|
||||
out = run(rank)
|
||||
assert out.num_nans is not None
|
||||
assert torch.equal(out.num_nans.cpu(), expected_nans)
|
||||
lp = out.logprobs_tensors
|
||||
assert lp is not None
|
||||
assert torch.equal(lp.logprob_token_ids.cpu(), expected_ids)
|
||||
assert torch.equal(lp.logprobs.cpu(), expected_logprobs)
|
||||
assert torch.equal(lp.selected_token_ranks.cpu(), expected_ranks)
|
||||
if num_logits != num_reqs:
|
||||
assert lp.cu_num_generated_tokens == cu.tolist()
|
||||
else:
|
||||
assert lp.cu_num_generated_tokens is None
|
||||
|
||||
|
||||
def test_finish_requests_frees_slots_in_sorted_order():
|
||||
"""Request slots must be freed in the same order on every TP rank:
|
||||
ownership derives from slot indices, and `finished_req_ids` is a set
|
||||
whose iteration order is per-process hash-randomized."""
|
||||
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
|
||||
|
||||
removed: list[str] = []
|
||||
fake_runner = SimpleNamespace(_remove_request=removed.append, pooling_runner=None)
|
||||
scheduler_output = SimpleNamespace(
|
||||
finished_req_ids={"req-b", "req-c", "req-a"},
|
||||
preempted_req_ids={"req-e", "req-d"},
|
||||
)
|
||||
GPUModelRunner.finish_requests(fake_runner, scheduler_output)
|
||||
assert removed == sorted(removed)
|
||||
assert set(removed) == {"req-a", "req-b", "req-c", "req-d", "req-e"}
|
||||
@@ -60,6 +60,7 @@ def _make_runner(**overrides: Any) -> Any:
|
||||
runner.parallel_config = SimpleNamespace(
|
||||
enable_eplb=True,
|
||||
enable_elastic_ep=False,
|
||||
enable_batch_sharded_sampling=False,
|
||||
eplb_config=SimpleNamespace(log_balancedness=True),
|
||||
)
|
||||
runner.vllm_config = SimpleNamespace(
|
||||
|
||||
@@ -164,6 +164,13 @@ class ParallelConfig:
|
||||
"""Whether the deployed model is MoE (if known)."""
|
||||
enable_expert_parallel: bool = False
|
||||
"""Use expert parallelism instead of tensor parallelism for MoE layers."""
|
||||
enable_batch_sharded_sampling: bool | None = None
|
||||
"""Use sharded sampling across tensor parallel ranks. Each rank samples
|
||||
a slice of the batch instead of every rank sampling all of it. Currently
|
||||
defaults to False if not set. Enabling it explicitly raises when the config
|
||||
cannot support it (`tensor_parallel_size` must be > 1, `max_num_seqs` at
|
||||
least `tensor_parallel_size`, and `max_logprobs` non-negative). Models opt in
|
||||
by implementing `compute_logits_local`."""
|
||||
enable_ep_weight_filter: bool = False
|
||||
"""Skip non-local expert weights during model loading when expert
|
||||
parallelism is active. Each rank only reads its own expert shard from
|
||||
|
||||
@@ -1675,6 +1675,8 @@ class VllmConfig:
|
||||
"Remove VLLM_USE_V2_MODEL_RUNNER=0."
|
||||
)
|
||||
|
||||
self._validate_batch_sharded_sampling()
|
||||
|
||||
# Re-compute compile ranges after platform-specific config updates
|
||||
# (e.g., XPU may lower max_num_batched_tokens when MLA is enabled)
|
||||
self._set_compile_ranges()
|
||||
@@ -2552,6 +2554,57 @@ class VllmConfig:
|
||||
|
||||
return unsupported
|
||||
|
||||
def _validate_batch_sharded_sampling(self) -> None:
|
||||
"""Validate `enable_batch_sharded_sampling` against the rest of the config."""
|
||||
if not self.parallel_config.enable_batch_sharded_sampling:
|
||||
# Default to False if not set.
|
||||
self.parallel_config.enable_batch_sharded_sampling = False
|
||||
return
|
||||
|
||||
blockers: list[str] = []
|
||||
tp_size = self.parallel_config.tensor_parallel_size
|
||||
|
||||
if not self.use_v2_model_runner:
|
||||
blockers.append("it is only implemented by Model Runner V2")
|
||||
|
||||
if tp_size <= 1:
|
||||
blockers.append("tensor_parallel_size is 1, so there is nothing to shard")
|
||||
elif self.scheduler_config.max_num_seqs < tp_size:
|
||||
# Requests are assigned to ranks whole, so fewer slots than ranks
|
||||
# leaves some ranks without work in every step.
|
||||
blockers.append(
|
||||
f"max_num_seqs ({self.scheduler_config.max_num_seqs}) is below "
|
||||
f"tensor_parallel_size ({tp_size})"
|
||||
)
|
||||
|
||||
if self.model_config is not None and self.model_config.max_logprobs < 0:
|
||||
# max_logprobs == -1 allows vocab-size logprob requests, which the
|
||||
# fixed-width logprobs gather cannot reasonably size for.
|
||||
blockers.append("max_logprobs is -1, allowing vocab-size logprob requests")
|
||||
|
||||
if (
|
||||
self.speculative_config is not None
|
||||
and self.speculative_config.enable_adaptive_verification
|
||||
):
|
||||
# Adaptive verification picks the per-request draft split on the GPU,
|
||||
# so cu_num_logits_np is only an upper bound, while the shard plan is
|
||||
# built from that CPU array. The two disagree once the budget binds.
|
||||
# TODO(TheEpicDolphin): Support adaptive verification with batch-sharded
|
||||
# sampling.
|
||||
blockers.append(
|
||||
"it does not yet work with adaptive verification, which decides "
|
||||
"the per-request logits counts on the GPU, where the CPU-side "
|
||||
"shard plan cannot see them"
|
||||
)
|
||||
|
||||
if blockers:
|
||||
raise ValueError(
|
||||
"Batch-sharded sampling was explicitly enabled via "
|
||||
"the --enable-batch-sharded-sampling flag, but is not supported "
|
||||
"in this configuration for the following reason(s): "
|
||||
f"{'; '.join(blockers)}."
|
||||
)
|
||||
|
||||
def _validate_v2_model_runner(self) -> None:
|
||||
"""Check for features not yet supported by the V2 model runner."""
|
||||
if not HAS_TRITON:
|
||||
|
||||
@@ -496,6 +496,9 @@ class EngineArgs:
|
||||
data_parallel_multi_port_external_lb: bool = False
|
||||
data_parallel_backend: DataParallelBackend = ParallelConfig.data_parallel_backend
|
||||
enable_expert_parallel: bool = ParallelConfig.enable_expert_parallel
|
||||
enable_batch_sharded_sampling: bool | None = (
|
||||
ParallelConfig.enable_batch_sharded_sampling
|
||||
)
|
||||
enable_ep_weight_filter: bool = ParallelConfig.enable_ep_weight_filter
|
||||
moe_backend: MoEBackend = KernelConfig.moe_backend
|
||||
linear_backend: LinearBackend = KernelConfig.linear_backend
|
||||
@@ -1156,6 +1159,10 @@ class EngineArgs:
|
||||
"-ep",
|
||||
**parallel_kwargs["enable_expert_parallel"],
|
||||
)
|
||||
parallel_group.add_argument(
|
||||
"--enable-batch-sharded-sampling",
|
||||
**parallel_kwargs["enable_batch_sharded_sampling"],
|
||||
)
|
||||
parallel_group.add_argument(
|
||||
"--enable-ep-weight-filter",
|
||||
**parallel_kwargs["enable_ep_weight_filter"],
|
||||
@@ -2279,6 +2286,7 @@ class EngineArgs:
|
||||
data_parallel_hybrid_lb=self.data_parallel_hybrid_lb,
|
||||
is_moe_model=model_config.is_moe,
|
||||
enable_expert_parallel=self.enable_expert_parallel,
|
||||
enable_batch_sharded_sampling=self.enable_batch_sharded_sampling,
|
||||
enable_ep_weight_filter=self.enable_ep_weight_filter,
|
||||
all2all_backend=self.all2all_backend,
|
||||
enable_elastic_ep=self.enable_elastic_ep,
|
||||
|
||||
@@ -155,7 +155,14 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
|
||||
hidden_states: torch.Tensor,
|
||||
lm_head: VocabParallelEmbedding,
|
||||
embedding_bias: torch.Tensor | None = None,
|
||||
skip_gather: bool = False,
|
||||
) -> torch.Tensor | None:
|
||||
# The LoRA delta is accumulated into the full gathered logits, so the
|
||||
# TP gather cannot be skipped here.
|
||||
if skip_gather:
|
||||
raise NotImplementedError(
|
||||
"Skipping the logits TP gather is not supported with an lm_head LoRA."
|
||||
)
|
||||
# Get the logits for the next tokens.
|
||||
if hasattr(lm_head, "base_layer"):
|
||||
actual_lm_head = lm_head.base_layer
|
||||
|
||||
@@ -99,12 +99,15 @@ class LogitsProcessor(PluggableLayer):
|
||||
lm_head: VocabParallelEmbedding,
|
||||
hidden_states: torch.Tensor,
|
||||
embedding_bias: torch.Tensor | None = None,
|
||||
skip_gather: bool = False,
|
||||
) -> torch.Tensor | None:
|
||||
if self.logits_as_input:
|
||||
logits = hidden_states
|
||||
else:
|
||||
# Get the logits for the next tokens.
|
||||
logits = self._get_logits(hidden_states, lm_head, embedding_bias)
|
||||
logits = self._get_logits(
|
||||
hidden_states, lm_head, embedding_bias, skip_gather
|
||||
)
|
||||
if logits is not None:
|
||||
if self.soft_cap is not None:
|
||||
logits = logits / self.soft_cap
|
||||
@@ -173,9 +176,12 @@ class LogitsProcessor(PluggableLayer):
|
||||
hidden_states: torch.Tensor,
|
||||
lm_head: VocabParallelEmbedding,
|
||||
embedding_bias: torch.Tensor | None,
|
||||
skip_gather: bool = False,
|
||||
) -> torch.Tensor | None:
|
||||
# Get the logits for the next tokens.
|
||||
logits = self._apply_head(lm_head, hidden_states, embedding_bias)
|
||||
if skip_gather:
|
||||
return logits
|
||||
|
||||
# Gather logits for TP
|
||||
if lm_head.tp_size > 1:
|
||||
|
||||
@@ -532,6 +532,12 @@ class LlamaForCausalLM(
|
||||
logits = self.logits_processor(self.lm_head, hidden_states)
|
||||
return logits
|
||||
|
||||
def compute_logits_local(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return self.logits_processor(self.lm_head, hidden_states, skip_gather=True)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
@@ -429,6 +429,12 @@ class Qwen3_5ForCausalLMBase(
|
||||
) -> torch.Tensor | None:
|
||||
return self.logits_processor(self.lm_head, hidden_states)
|
||||
|
||||
def compute_logits_local(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return self.logits_processor(self.lm_head, hidden_states, skip_gather=True)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
@@ -591,6 +597,9 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid)
|
||||
|
||||
return hidden_states
|
||||
|
||||
def compute_logits_local(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
return self.language_model.compute_logits_local(hidden_states)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -1765,6 +1765,12 @@ class DeepseekV4ForCausalLM(
|
||||
logits = self.logits_processor(self.lm_head, hidden_states)
|
||||
return logits
|
||||
|
||||
def compute_logits_local(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return self.logits_processor(self.lm_head, hidden_states, skip_gather=True)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
|
||||
@@ -63,6 +63,7 @@ class InklingLogitsProcessor(LogitsProcessor):
|
||||
lm_head: VocabParallelEmbedding,
|
||||
hidden_states: torch.Tensor,
|
||||
embedding_bias: torch.Tensor | None = None,
|
||||
skip_gather: bool = False,
|
||||
) -> torch.Tensor | None:
|
||||
# ``base_layer`` exists only on the LogitsProcessorWithLoRA wrapper,
|
||||
# which calls this forward with ``self`` bound to the wrapper. The
|
||||
@@ -72,15 +73,16 @@ class InklingLogitsProcessor(LogitsProcessor):
|
||||
# lives on this class).
|
||||
if hasattr(self, "base_layer"):
|
||||
return type(self.base_layer)._lora_forward(
|
||||
self, lm_head, hidden_states, embedding_bias
|
||||
self, lm_head, hidden_states, embedding_bias, skip_gather
|
||||
)
|
||||
return self._base_forward(lm_head, hidden_states, embedding_bias)
|
||||
return self._base_forward(lm_head, hidden_states, embedding_bias, skip_gather)
|
||||
|
||||
def _lora_forward(
|
||||
self,
|
||||
lm_head: VocabParallelEmbedding,
|
||||
hidden_states: torch.Tensor,
|
||||
embedding_bias: torch.Tensor | None = None,
|
||||
skip_gather: bool = False,
|
||||
) -> torch.Tensor | None:
|
||||
# ``self`` is the LogitsProcessorWithLoRA wrapper here: ``_get_logits``
|
||||
# returns the base logits plus the lm_head LoRA delta. Apply the muP
|
||||
@@ -90,7 +92,9 @@ class InklingLogitsProcessor(LogitsProcessor):
|
||||
if self.logits_as_input:
|
||||
logits = hidden_states
|
||||
else:
|
||||
logits = self._get_logits(hidden_states, lm_head, embedding_bias)
|
||||
logits = self._get_logits(
|
||||
hidden_states, lm_head, embedding_bias, skip_gather
|
||||
)
|
||||
# TODO: fuse this multiplication
|
||||
if logits is not None and mup:
|
||||
assert self.base_layer.soft_cap is None
|
||||
@@ -103,10 +107,11 @@ class InklingLogitsProcessor(LogitsProcessor):
|
||||
lm_head: VocabParallelEmbedding,
|
||||
hidden_states: torch.Tensor,
|
||||
embedding_bias: torch.Tensor | None = None,
|
||||
skip_gather: bool = False,
|
||||
) -> torch.Tensor | None:
|
||||
mup = self.logits_mup_width_multiplier
|
||||
if not mup:
|
||||
return super().forward(lm_head, hidden_states, embedding_bias)
|
||||
return super().forward(lm_head, hidden_states, embedding_bias, skip_gather)
|
||||
# Fold the muP width divisor into the lm_head GEMM alpha (fp32 epilogue):
|
||||
# no separate elementwise kernel, no bf16 rounding of scaled logits, and
|
||||
# no weight mutation. Overfit to the served checkpoint: bf16 lm_head, no
|
||||
@@ -123,6 +128,8 @@ class InklingLogitsProcessor(LogitsProcessor):
|
||||
beta=0.0,
|
||||
alpha=1.0 / mup,
|
||||
)
|
||||
if skip_gather:
|
||||
return logits
|
||||
logits = self._gather_logits(logits)
|
||||
if logits is not None:
|
||||
logits = logits[..., : self.org_vocab_size]
|
||||
|
||||
@@ -63,6 +63,7 @@ class InklingLogitsProcessor(LogitsProcessor):
|
||||
lm_head: VocabParallelEmbedding,
|
||||
hidden_states: torch.Tensor,
|
||||
embedding_bias: torch.Tensor | None = None,
|
||||
skip_gather: bool = False,
|
||||
) -> torch.Tensor | None:
|
||||
# ``base_layer`` exists only on the LogitsProcessorWithLoRA wrapper,
|
||||
# which calls this forward with ``self`` bound to the wrapper. The
|
||||
@@ -72,15 +73,16 @@ class InklingLogitsProcessor(LogitsProcessor):
|
||||
# lives on this class).
|
||||
if hasattr(self, "base_layer"):
|
||||
return type(self.base_layer)._lora_forward(
|
||||
self, lm_head, hidden_states, embedding_bias
|
||||
self, lm_head, hidden_states, embedding_bias, skip_gather
|
||||
)
|
||||
return self._base_forward(lm_head, hidden_states, embedding_bias)
|
||||
return self._base_forward(lm_head, hidden_states, embedding_bias, skip_gather)
|
||||
|
||||
def _lora_forward(
|
||||
self,
|
||||
lm_head: VocabParallelEmbedding,
|
||||
hidden_states: torch.Tensor,
|
||||
embedding_bias: torch.Tensor | None = None,
|
||||
skip_gather: bool = False,
|
||||
) -> torch.Tensor | None:
|
||||
# ``self`` is the LogitsProcessorWithLoRA wrapper here: ``_get_logits``
|
||||
# returns the base logits plus the lm_head LoRA delta. Apply the muP
|
||||
@@ -90,7 +92,9 @@ class InklingLogitsProcessor(LogitsProcessor):
|
||||
if self.logits_as_input:
|
||||
logits = hidden_states
|
||||
else:
|
||||
logits = self._get_logits(hidden_states, lm_head, embedding_bias)
|
||||
logits = self._get_logits(
|
||||
hidden_states, lm_head, embedding_bias, skip_gather
|
||||
)
|
||||
# TODO: fuse this multiplication
|
||||
if logits is not None and mup:
|
||||
assert self.base_layer.soft_cap is None
|
||||
@@ -103,10 +107,11 @@ class InklingLogitsProcessor(LogitsProcessor):
|
||||
lm_head: VocabParallelEmbedding,
|
||||
hidden_states: torch.Tensor,
|
||||
embedding_bias: torch.Tensor | None = None,
|
||||
skip_gather: bool = False,
|
||||
) -> torch.Tensor | None:
|
||||
mup = self.logits_mup_width_multiplier
|
||||
if not mup:
|
||||
return super().forward(lm_head, hidden_states, embedding_bias)
|
||||
return super().forward(lm_head, hidden_states, embedding_bias, skip_gather)
|
||||
# Fold the muP width divisor into the lm_head GEMM alpha (fp32 epilogue):
|
||||
# no separate elementwise kernel, no bf16 rounding of scaled logits, and
|
||||
# no weight mutation. Overfit to the served checkpoint: bf16 lm_head, no
|
||||
@@ -123,6 +128,8 @@ class InklingLogitsProcessor(LogitsProcessor):
|
||||
beta=0.0,
|
||||
alpha=1.0 / mup,
|
||||
)
|
||||
if skip_gather:
|
||||
return logits
|
||||
logits = self._gather_logits(logits)
|
||||
if logits is not None:
|
||||
logits = logits[..., : self.org_vocab_size]
|
||||
|
||||
@@ -1059,6 +1059,9 @@ class MiniMaxM3SparseForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
|
||||
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
|
||||
return self.logits_processor(self.lm_head, hidden_states)
|
||||
|
||||
def compute_logits_local(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
return self.logits_processor(self.lm_head, hidden_states, skip_gather=True)
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return self.model.get_expert_mapping()
|
||||
|
||||
@@ -1272,6 +1275,9 @@ class MiniMaxM3SparseForConditionalGeneration(
|
||||
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
|
||||
return self.language_model.compute_logits(hidden_states)
|
||||
|
||||
def compute_logits_local(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
return self.language_model.compute_logits_local(hidden_states)
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return self.language_model.get_expert_mapping()
|
||||
|
||||
|
||||
@@ -127,6 +127,11 @@ from vllm.v1.worker.gpu.mm.lora import set_active_mm_loras
|
||||
from vllm.v1.worker.gpu.model_states import init_model_state
|
||||
from vllm.v1.worker.gpu.pool.pooling_runner import PoolingRunner
|
||||
from vllm.v1.worker.gpu.pp_utils import PPHandler
|
||||
from vllm.v1.worker.gpu.sample.batch_shard import (
|
||||
BatchSharder,
|
||||
all_to_all_logits,
|
||||
gather_sampler_output,
|
||||
)
|
||||
from vllm.v1.worker.gpu.sample.output import SamplerOutput
|
||||
from vllm.v1.worker.gpu.sample.prompt_logprob import PromptLogprobsWorker
|
||||
from vllm.v1.worker.gpu.sample.sampler import Sampler
|
||||
@@ -310,6 +315,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# model_state exists (num_new_sampled_tokens_per_step from ModelState).
|
||||
self.sampler: Sampler | None = None
|
||||
self.rejection_sampler: RejectionSampler | None = None
|
||||
self.batch_sharder: BatchSharder | None = None
|
||||
self.prompt_logprobs_worker: PromptLogprobsWorker | None = None
|
||||
self.structured_outputs_worker: StructuredOutputsWorker | None = None
|
||||
self.cudagraph_manager: ModelCudaGraphManager | None = None
|
||||
@@ -404,6 +410,21 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
+ self.model_state.num_new_sampled_tokens_per_step
|
||||
)
|
||||
|
||||
if self.parallel_config.enable_batch_sharded_sampling:
|
||||
if hasattr(self.model, "compute_logits_local"):
|
||||
self.batch_sharder = BatchSharder(
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
max_num_logits_per_req=self.decode_query_len,
|
||||
device=self.device,
|
||||
)
|
||||
logger.info("Batch-sharded sampling enabled.")
|
||||
else:
|
||||
logger.warning_once(
|
||||
"Disabling batch-sharded sampling: %s does not implement "
|
||||
"compute_logits_local",
|
||||
type(self.model).__name__,
|
||||
)
|
||||
|
||||
# Initialize samplers. Model states may override via custom_sampler().
|
||||
if self.is_last_pp_rank and not self.is_pooling_model:
|
||||
self.sampler = Sampler(
|
||||
@@ -935,7 +956,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
preempted_req_ids = scheduler_output.preempted_req_ids
|
||||
if preempted_req_ids:
|
||||
finished_req_ids = finished_req_ids.union(preempted_req_ids)
|
||||
for req_id in finished_req_ids:
|
||||
# Sorted so every TP rank frees request slots in the same order.
|
||||
# Features like batch-sharded sampling derive rank request ownership
|
||||
# from the slot index.
|
||||
for req_id in sorted(finished_req_ids):
|
||||
self._remove_request(req_id)
|
||||
|
||||
def free_states(self, scheduler_output: SchedulerOutput) -> None:
|
||||
@@ -1335,8 +1359,25 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
input_batch: InputBatch,
|
||||
grammar_output: GrammarOutput | None,
|
||||
) -> tuple[SamplerOutput, torch.Tensor, torch.Tensor]:
|
||||
sample_hidden_states = hidden_states[input_batch.logits_indices]
|
||||
logits = self.model.compute_logits(sample_hidden_states)
|
||||
shard_metadata = None
|
||||
global_input_batch = input_batch
|
||||
if self.batch_sharder is not None:
|
||||
# Shard the inputs along the batch dimension to sample in parallel
|
||||
# across TP ranks.
|
||||
input_batch, sorted_logits_indices, grammar_output, shard_metadata = (
|
||||
self.batch_sharder.shard_sampler_inputs(input_batch, grammar_output)
|
||||
)
|
||||
# The hidden states must be gathered in rank-owner-sorted order
|
||||
# before computing the partial-vocab logits, so that the all-to-all
|
||||
# produces full-vocab logits for just the locally-owned requests.
|
||||
sample_hidden_states = hidden_states[sorted_logits_indices]
|
||||
local_logits = self.model.compute_logits_local(sample_hidden_states)
|
||||
logits = all_to_all_logits(local_logits, shard_metadata)
|
||||
logits = logits[:, : self.vocab_size]
|
||||
else:
|
||||
sample_hidden_states = hidden_states[input_batch.logits_indices]
|
||||
logits = self.model.compute_logits(sample_hidden_states)
|
||||
|
||||
if grammar_output is not None:
|
||||
# Apply grammar bitmask to the logits in-place.
|
||||
assert self.structured_outputs_worker is not None
|
||||
@@ -1347,7 +1388,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
grammar_output.grammar_bitmask,
|
||||
)
|
||||
|
||||
if input_batch.num_draft_tokens == 0 or self.rejection_sampler is None:
|
||||
sampler_output: SamplerOutput | None
|
||||
if input_batch.num_reqs == 0:
|
||||
# This rank owns no requests this step. It contributes an
|
||||
# all-padding block to the gather below.
|
||||
sampler_output = None
|
||||
elif input_batch.num_draft_tokens == 0 or self.rejection_sampler is None:
|
||||
assert self.sampler is not None
|
||||
sampler_output = self.sampler(logits, input_batch)
|
||||
else:
|
||||
@@ -1361,6 +1407,28 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.speculator.draft_logits,
|
||||
)
|
||||
|
||||
if shard_metadata is not None:
|
||||
# Gather the sharded sampler outputs from the TP ranks into a single
|
||||
# sampler output.
|
||||
assert self.sampler is not None
|
||||
sampler_output = gather_sampler_output(
|
||||
sampler_output,
|
||||
shard_metadata,
|
||||
device=self.device,
|
||||
global_batch=global_input_batch,
|
||||
local_batch=input_batch,
|
||||
gather_num_nans=self.sampler.compute_nans,
|
||||
logprobs_dims=self.sampler.get_logprobs_dims(
|
||||
global_input_batch.idx_mapping_np,
|
||||
# Rejection sampler does not return logprob token ids.
|
||||
include_token_ids=(
|
||||
global_input_batch.num_draft_tokens == 0
|
||||
or self.rejection_sampler is None
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
assert sampler_output is not None
|
||||
return sampler_output, sampler_output.num_sampled, sampler_output.num_rejected
|
||||
|
||||
def postprocess_sampled(
|
||||
|
||||
@@ -0,0 +1,674 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass, replace
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.distributed
|
||||
|
||||
from vllm.distributed.communication_op import tensor_model_parallel_all_gather
|
||||
from vllm.distributed.parallel_state import get_tp_group
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.core.sched.output import GrammarOutput
|
||||
from vllm.v1.outputs import LogprobsTensors
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.sample.output import SamplerOutput
|
||||
|
||||
|
||||
@dataclass
|
||||
class BatchShardMetadata:
|
||||
"""Collective layout for one sampling step.
|
||||
|
||||
Requests (and their logits) are owner-sorted: all requests owned by
|
||||
rank 0 first, then rank 1, etc. Within an owner, batch order is
|
||||
preserved (stable sort). Every field is a pure function of the
|
||||
replicated idx_mapping and cu_num_logits, so all ranks build identical
|
||||
plans without communication — which relies on request slots
|
||||
(idx_mapping) being assigned identically on every rank (see
|
||||
finish_requests in the model runner).
|
||||
"""
|
||||
|
||||
tp_size: int
|
||||
# Per-owner-rank logits counts (all-to-all send splits).
|
||||
num_logits_per_rank: list[int]
|
||||
num_local_logits: int
|
||||
num_local_reqs: int
|
||||
# The number of request entries contributed by each rank to the
|
||||
# gathered result (shorter shards are padded up to this value).
|
||||
max_num_reqs_per_rank: int
|
||||
# [num_reqs] For batch request i, its source index in the rank-major
|
||||
# gathered results. Indexing the gathered tensors with this restores
|
||||
# the original batch order.
|
||||
gathered_src_indices: torch.Tensor
|
||||
# The number of entries occupied by each request in the gathered
|
||||
# tensor. Derived from the global cu_num_logits, so each rank agrees
|
||||
# ont he gather shapes, including empty shards.
|
||||
max_num_logits_per_req: int
|
||||
|
||||
|
||||
@triton.jit(
|
||||
do_not_specialize=["num_reqs", "local_logits_start", "max_num_reqs_per_rank"]
|
||||
)
|
||||
def _build_shard_plan_kernel(
|
||||
idx_mapping_ptr,
|
||||
cu_num_logits_ptr,
|
||||
query_start_loc_ptr,
|
||||
seq_lens_ptr,
|
||||
sorted_logits_indices_ptr,
|
||||
gathered_src_indices_ptr,
|
||||
local_idx_mapping_ptr,
|
||||
local_cu_num_logits_ptr,
|
||||
local_logits_indices_ptr,
|
||||
local_expanded_idx_mapping_ptr,
|
||||
local_expanded_local_pos_ptr,
|
||||
local_seq_lens_ptr,
|
||||
num_reqs,
|
||||
local_logits_start,
|
||||
max_num_reqs_per_rank,
|
||||
TP_SIZE: tl.constexpr,
|
||||
TP_RANK: tl.constexpr,
|
||||
PADDED_NUM_REQS: tl.constexpr,
|
||||
PADDED_NUM_LOGITS_PER_REQ: tl.constexpr,
|
||||
):
|
||||
req_idx = tl.program_id(0)
|
||||
req_state_idx = tl.load(idx_mapping_ptr + req_idx)
|
||||
owner = req_state_idx % TP_SIZE
|
||||
num_logits = tl.load(cu_num_logits_ptr + req_idx + 1) - tl.load(
|
||||
cu_num_logits_ptr + req_idx
|
||||
)
|
||||
|
||||
req_block = tl.arange(0, PADDED_NUM_REQS)
|
||||
req_mask = req_block < num_reqs
|
||||
owners = tl.load(idx_mapping_ptr + req_block, mask=req_mask, other=0) % TP_SIZE
|
||||
num_logits_per_req = tl.load(
|
||||
cu_num_logits_ptr + req_block + 1, mask=req_mask, other=0
|
||||
) - tl.load(cu_num_logits_ptr + req_block, mask=req_mask, other=0)
|
||||
|
||||
# Owner-sorted layout is rank 0's requests in batch order, then rank 1's,
|
||||
# etc, so a request's starting offset can be derived from counting.
|
||||
lower = (owners < owner) & req_mask
|
||||
earlier = (owners == owner) & (req_block < req_idx) & req_mask
|
||||
|
||||
# Compute the logits row offset for the current request in the sorted
|
||||
# batch.
|
||||
req_logits_start = tl.sum(tl.where(lower, num_logits_per_req, 0)) + tl.sum(
|
||||
tl.where(earlier, num_logits_per_req, 0)
|
||||
)
|
||||
|
||||
# Copy the current request logit indices into the sorted batch.
|
||||
query_start = tl.load(query_start_loc_ptr + req_idx + 1) - num_logits
|
||||
logit_block = tl.arange(0, PADDED_NUM_LOGITS_PER_REQ)
|
||||
logit_mask = logit_block < num_logits
|
||||
logits_indices = (query_start + logit_block).to(tl.int64)
|
||||
tl.store(
|
||||
sorted_logits_indices_ptr + req_logits_start + logit_block,
|
||||
logits_indices,
|
||||
mask=logit_mask,
|
||||
)
|
||||
|
||||
# Write the mapping of output tensor request indices => gathered tensor
|
||||
# request indices.
|
||||
# Batch request i's result lives in the padded, gathered results at
|
||||
# rank_of(i) * max_num_reqs_per_rank + offset_within_rank(i).
|
||||
local_req_idx = tl.sum(earlier.to(tl.int32))
|
||||
tl.store(
|
||||
gathered_src_indices_ptr + req_idx,
|
||||
(owner * max_num_reqs_per_rank + local_req_idx).to(tl.int64),
|
||||
)
|
||||
|
||||
if req_idx == 0:
|
||||
tl.store(local_cu_num_logits_ptr, 0)
|
||||
|
||||
if owner == TP_RANK:
|
||||
local_start = req_logits_start - local_logits_start
|
||||
tl.store(local_idx_mapping_ptr + local_req_idx, req_state_idx)
|
||||
tl.store(local_cu_num_logits_ptr + local_req_idx + 1, local_start + num_logits)
|
||||
tl.store(
|
||||
local_logits_indices_ptr + local_start + logit_block,
|
||||
logits_indices,
|
||||
mask=logit_mask,
|
||||
)
|
||||
tl.store(
|
||||
local_expanded_idx_mapping_ptr + local_start + logit_block,
|
||||
req_state_idx,
|
||||
mask=logit_mask,
|
||||
)
|
||||
tl.store(
|
||||
local_expanded_local_pos_ptr + local_start + logit_block,
|
||||
logit_block,
|
||||
mask=logit_mask,
|
||||
)
|
||||
tl.store(local_seq_lens_ptr + local_req_idx, tl.load(seq_lens_ptr + req_idx))
|
||||
|
||||
|
||||
class BatchSharder:
|
||||
"""Shards the sampler inputs across TP ranks along the batch dimension."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
max_num_reqs: int,
|
||||
max_num_logits_per_req: int,
|
||||
device: torch.device,
|
||||
):
|
||||
tp_group = get_tp_group()
|
||||
self.tp_rank = tp_group.rank_in_group
|
||||
self.tp_size = tp_group.world_size
|
||||
self.device = device
|
||||
self._padded_num_reqs = triton.next_power_of_2(max_num_reqs)
|
||||
self._padded_num_logits_per_req = triton.next_power_of_2(max_num_logits_per_req)
|
||||
self._num_warps = max(1, min(8, self._padded_num_reqs // 128))
|
||||
|
||||
def shard_sampler_inputs(
|
||||
self,
|
||||
input_batch: InputBatch,
|
||||
grammar_output: GrammarOutput | None,
|
||||
) -> tuple[InputBatch, torch.Tensor, GrammarOutput | None, BatchShardMetadata]:
|
||||
"""Owner-sort the batch and build this rank's local sampler inputs.
|
||||
|
||||
Returns the local sub-batch, the owner-sorted logits_indices (gather
|
||||
the sampling hidden states with these so `compute_logits_local` emits
|
||||
logits in all-to-all send order), the local grammar output (None if
|
||||
this rank owns none of the structured-output requests), and the shard
|
||||
metadata (used for all-gathering the sampler outputs).
|
||||
"""
|
||||
tp_rank = self.tp_rank
|
||||
tp_size = self.tp_size
|
||||
num_reqs = input_batch.idx_mapping_np.shape[0]
|
||||
num_logits = int(input_batch.cu_num_logits_np[-1])
|
||||
|
||||
# Deterministically assign requests to ranks, round-robin over slot
|
||||
# indices so ownership stays balanced when the slot allocator fills
|
||||
# low slots first (partial occupancy).
|
||||
# NOTE: This mirrors the assignment used in _build_shard_plan_kernel.
|
||||
req_owner_np = input_batch.idx_mapping_np % tp_size
|
||||
local_req_indices_np = np.flatnonzero(req_owner_np == tp_rank)
|
||||
local_idx_mapping_np = input_batch.idx_mapping_np[local_req_indices_np]
|
||||
num_local_reqs = local_req_indices_np.shape[0]
|
||||
num_reqs_per_rank_np = np.bincount(req_owner_np, minlength=tp_size)
|
||||
max_num_reqs_per_rank = int(num_reqs_per_rank_np.max()) if num_reqs else 1
|
||||
# Derive the number of logits owned by each rank, as well as the
|
||||
# local rank.
|
||||
num_logits_per_req_np = np.diff(input_batch.cu_num_logits_np)
|
||||
num_logits_per_rank_np = np.bincount(
|
||||
req_owner_np, weights=num_logits_per_req_np, minlength=tp_size
|
||||
).astype(np.int64)
|
||||
num_local_logits = int(num_logits_per_rank_np[tp_rank])
|
||||
local_logits_start = int(num_logits_per_rank_np[:tp_rank].sum())
|
||||
local_cu_num_logits_np = np.zeros(num_local_reqs + 1, dtype=np.int32)
|
||||
np.cumsum(
|
||||
num_logits_per_req_np[local_req_indices_np], out=local_cu_num_logits_np[1:]
|
||||
)
|
||||
max_num_logits_per_req = int(num_logits_per_req_np.max()) if num_reqs else 1
|
||||
|
||||
# Shard the input batch GPU tensors.
|
||||
sorted_logits_indices = torch.empty(
|
||||
num_logits, dtype=torch.int64, device=self.device
|
||||
)
|
||||
gathered_src_indices = torch.empty(
|
||||
num_reqs, dtype=torch.int64, device=self.device
|
||||
)
|
||||
local_logits_indices = torch.empty(
|
||||
num_local_logits, dtype=torch.int64, device=self.device
|
||||
)
|
||||
local_idx_mapping = torch.empty(
|
||||
num_local_reqs, dtype=torch.int32, device=self.device
|
||||
)
|
||||
local_cu_num_logits = torch.empty(
|
||||
num_local_reqs + 1, dtype=torch.int32, device=self.device
|
||||
)
|
||||
local_expanded_idx_mapping = torch.empty(
|
||||
num_local_logits, dtype=torch.int32, device=self.device
|
||||
)
|
||||
local_expanded_local_pos = torch.empty(
|
||||
num_local_logits, dtype=torch.int32, device=self.device
|
||||
)
|
||||
local_seq_lens = torch.empty(
|
||||
num_local_reqs, dtype=torch.int32, device=self.device
|
||||
)
|
||||
if num_reqs > 0:
|
||||
_build_shard_plan_kernel[(num_reqs,)](
|
||||
input_batch.idx_mapping,
|
||||
input_batch.cu_num_logits,
|
||||
input_batch.query_start_loc,
|
||||
input_batch.seq_lens,
|
||||
sorted_logits_indices,
|
||||
gathered_src_indices,
|
||||
local_idx_mapping,
|
||||
local_cu_num_logits,
|
||||
local_logits_indices,
|
||||
local_expanded_idx_mapping,
|
||||
local_expanded_local_pos,
|
||||
local_seq_lens,
|
||||
num_reqs,
|
||||
local_logits_start,
|
||||
max_num_reqs_per_rank,
|
||||
TP_SIZE=tp_size,
|
||||
TP_RANK=tp_rank,
|
||||
PADDED_NUM_REQS=self._padded_num_reqs,
|
||||
PADDED_NUM_LOGITS_PER_REQ=self._padded_num_logits_per_req,
|
||||
num_warps=self._num_warps,
|
||||
)
|
||||
|
||||
# Compute the local number of draft tokens.
|
||||
num_draft_tokens_per_req = None
|
||||
num_draft_tokens = 0
|
||||
if input_batch.num_draft_tokens_per_req is not None:
|
||||
num_draft_tokens_per_req = input_batch.num_draft_tokens_per_req[
|
||||
local_req_indices_np
|
||||
]
|
||||
num_draft_tokens = int(num_draft_tokens_per_req.sum())
|
||||
|
||||
local_req_ids = [input_batch.req_ids[i] for i in local_req_indices_np.tolist()]
|
||||
local_batch = replace(
|
||||
input_batch,
|
||||
req_ids=local_req_ids,
|
||||
num_reqs=num_local_reqs,
|
||||
idx_mapping=local_idx_mapping,
|
||||
idx_mapping_np=local_idx_mapping_np,
|
||||
expanded_idx_mapping=local_expanded_idx_mapping,
|
||||
expanded_local_pos=local_expanded_local_pos,
|
||||
seq_lens=local_seq_lens,
|
||||
logits_indices=local_logits_indices,
|
||||
cu_num_logits=local_cu_num_logits,
|
||||
cu_num_logits_np=local_cu_num_logits_np,
|
||||
num_draft_tokens=num_draft_tokens,
|
||||
num_draft_tokens_per_req=num_draft_tokens_per_req,
|
||||
)
|
||||
local_grammar_output = None
|
||||
if grammar_output is not None:
|
||||
local_grammar_output = _shard_grammar_output(
|
||||
grammar_output, input_batch, local_req_ids
|
||||
)
|
||||
metadata = BatchShardMetadata(
|
||||
tp_size=tp_size,
|
||||
num_logits_per_rank=num_logits_per_rank_np.tolist(),
|
||||
num_local_logits=num_local_logits,
|
||||
num_local_reqs=num_local_reqs,
|
||||
max_num_reqs_per_rank=max_num_reqs_per_rank,
|
||||
gathered_src_indices=gathered_src_indices,
|
||||
max_num_logits_per_req=max_num_logits_per_req,
|
||||
)
|
||||
return local_batch, sorted_logits_indices, local_grammar_output, metadata
|
||||
|
||||
|
||||
def all_to_all_logits(
|
||||
logits_shard: torch.Tensor,
|
||||
metadata: BatchShardMetadata,
|
||||
) -> torch.Tensor:
|
||||
recv = logits_shard.new_empty(
|
||||
metadata.tp_size * metadata.num_local_logits, logits_shard.shape[-1]
|
||||
)
|
||||
torch.distributed.all_to_all_single(
|
||||
recv,
|
||||
logits_shard.contiguous(),
|
||||
output_split_sizes=[metadata.num_local_logits] * metadata.tp_size,
|
||||
input_split_sizes=metadata.num_logits_per_rank,
|
||||
group=get_tp_group().device_group,
|
||||
)
|
||||
shard_width = recv.shape[-1]
|
||||
return (
|
||||
recv.view(metadata.tp_size, metadata.num_local_logits, shard_width)
|
||||
.permute(1, 0, 2)
|
||||
.reshape(metadata.num_local_logits, metadata.tp_size * shard_width)
|
||||
)
|
||||
|
||||
|
||||
def _shard_grammar_output(
|
||||
grammar_output: GrammarOutput,
|
||||
input_batch: InputBatch,
|
||||
local_req_ids: list[str],
|
||||
) -> GrammarOutput | None:
|
||||
owned = set(local_req_ids)
|
||||
req_id_to_idx = {req_id: i for i, req_id in enumerate(input_batch.req_ids)}
|
||||
cu_num_logits_np = input_batch.cu_num_logits_np
|
||||
|
||||
local_ids: list[str] = []
|
||||
keep_indices: list[int] = []
|
||||
cursor = 0
|
||||
for req_id in grammar_output.structured_output_request_ids:
|
||||
req_idx = req_id_to_idx[req_id]
|
||||
num_req_logits = int(cu_num_logits_np[req_idx + 1] - cu_num_logits_np[req_idx])
|
||||
if req_id in owned:
|
||||
local_ids.append(req_id)
|
||||
keep_indices.extend(range(cursor, cursor + num_req_logits))
|
||||
cursor += num_req_logits
|
||||
if not local_ids:
|
||||
return None
|
||||
return GrammarOutput(
|
||||
structured_output_request_ids=local_ids,
|
||||
grammar_bitmask=grammar_output.grammar_bitmask[keep_indices],
|
||||
)
|
||||
|
||||
|
||||
@triton.jit(
|
||||
do_not_specialize=["max_num_logits_per_req", "num_src_cols", "packed_stride"]
|
||||
)
|
||||
def _pack_sampler_output_kernel(
|
||||
packed_ptr,
|
||||
packed_stride,
|
||||
sampled_token_ids_ptr,
|
||||
sampled_token_ids_stride,
|
||||
num_sampled_ptr,
|
||||
num_rejected_ptr,
|
||||
num_nans_ptr,
|
||||
local_cu_num_logits_ptr,
|
||||
max_num_logits_per_req,
|
||||
num_src_cols,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
req_idx = tl.program_id(0)
|
||||
cols = tl.arange(0, BLOCK_SIZE)
|
||||
row_ptr = packed_ptr + req_idx * packed_stride
|
||||
|
||||
token_ids = tl.load(
|
||||
sampled_token_ids_ptr + req_idx * sampled_token_ids_stride + cols,
|
||||
mask=cols < num_src_cols,
|
||||
other=0,
|
||||
)
|
||||
tl.store(row_ptr + cols, token_ids, mask=cols < max_num_logits_per_req)
|
||||
num_sampled = tl.load(num_sampled_ptr + req_idx)
|
||||
tl.store(row_ptr + max_num_logits_per_req, num_sampled.to(tl.int64))
|
||||
num_rejected = tl.load(num_rejected_ptr + req_idx)
|
||||
tl.store(row_ptr + max_num_logits_per_req + 1, num_rejected.to(tl.int64))
|
||||
if num_nans_ptr is not None:
|
||||
# num_nans is per logits row; reduce it per request. The replicated
|
||||
# samplers report per-row counts that downstream zips per request,
|
||||
# so a per-request sum is the well-defined equivalent.
|
||||
start = tl.load(local_cu_num_logits_ptr + req_idx)
|
||||
num_req_logits = tl.load(local_cu_num_logits_ptr + req_idx + 1) - start
|
||||
nans = tl.load(num_nans_ptr + start + cols, mask=cols < num_req_logits, other=0)
|
||||
tl.store(
|
||||
row_ptr + max_num_logits_per_req + 2, tl.sum(nans.to(tl.int64), axis=0)
|
||||
)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["max_num_logits_per_req", "gathered_stride"])
|
||||
def _unpack_gathered_output_kernel(
|
||||
gathered_ptr,
|
||||
gathered_stride,
|
||||
gathered_src_indices_ptr,
|
||||
sampled_token_ids_ptr,
|
||||
num_sampled_ptr,
|
||||
num_rejected_ptr,
|
||||
num_nans_ptr,
|
||||
max_num_logits_per_req,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
req_idx = tl.program_id(0)
|
||||
src = tl.load(gathered_src_indices_ptr + req_idx)
|
||||
row_ptr = gathered_ptr + src * gathered_stride
|
||||
cols = tl.arange(0, BLOCK_SIZE)
|
||||
mask = cols < max_num_logits_per_req
|
||||
|
||||
token_ids = tl.load(row_ptr + cols, mask=mask)
|
||||
tl.store(
|
||||
sampled_token_ids_ptr + req_idx * max_num_logits_per_req + cols,
|
||||
token_ids,
|
||||
mask=mask,
|
||||
)
|
||||
num_sampled = tl.load(row_ptr + max_num_logits_per_req)
|
||||
tl.store(num_sampled_ptr + req_idx, num_sampled.to(tl.int32))
|
||||
num_rejected = tl.load(row_ptr + max_num_logits_per_req + 1)
|
||||
tl.store(num_rejected_ptr + req_idx, num_rejected.to(tl.int32))
|
||||
if num_nans_ptr is not None:
|
||||
num_nans = tl.load(row_ptr + max_num_logits_per_req + 2)
|
||||
tl.store(num_nans_ptr + req_idx, num_nans.to(tl.int32))
|
||||
|
||||
|
||||
@triton.jit(
|
||||
do_not_specialize=[
|
||||
"max_num_logits_per_req",
|
||||
"num_logprob_cols",
|
||||
"num_src_cols",
|
||||
"send_ids_stride",
|
||||
"send_logprobs_stride",
|
||||
"src_ids_stride",
|
||||
"src_logprobs_stride",
|
||||
]
|
||||
)
|
||||
def _pack_logprobs_kernel(
|
||||
send_ids_ptr,
|
||||
send_ids_stride,
|
||||
send_logprobs_ptr,
|
||||
send_logprobs_stride,
|
||||
src_ids_ptr,
|
||||
src_ids_stride,
|
||||
src_logprobs_ptr,
|
||||
src_logprobs_stride,
|
||||
selected_token_ranks_ptr,
|
||||
local_cu_num_logits_ptr,
|
||||
max_num_logits_per_req,
|
||||
num_logprob_cols,
|
||||
num_src_cols,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
req_idx = tl.program_id(0)
|
||||
pos = tl.program_id(1)
|
||||
start = tl.load(local_cu_num_logits_ptr + req_idx)
|
||||
if pos < tl.load(local_cu_num_logits_ptr + req_idx + 1) - start:
|
||||
src = start + pos
|
||||
dst = req_idx * max_num_logits_per_req + pos
|
||||
cols = tl.arange(0, BLOCK_SIZE)
|
||||
# A rank whose sub-batch has no draft tokens runs the regular sampler
|
||||
# while others run the rejection sampler, and their column counts can
|
||||
# differ (e.g. logprob_token_ids widening); the two masks pad or
|
||||
# truncate to the agreed width.
|
||||
src_mask = cols < num_src_cols
|
||||
dst_mask = cols < num_logprob_cols
|
||||
ids = tl.load(src_ids_ptr + src * src_ids_stride + cols, mask=src_mask, other=0)
|
||||
logprobs = tl.load(
|
||||
src_logprobs_ptr + src * src_logprobs_stride + cols,
|
||||
mask=src_mask,
|
||||
other=float("-inf"),
|
||||
)
|
||||
ids_row = send_ids_ptr + dst * send_ids_stride
|
||||
tl.store(ids_row + cols, ids.to(tl.int64), mask=dst_mask)
|
||||
rank = tl.load(selected_token_ranks_ptr + src)
|
||||
tl.store(ids_row + num_logprob_cols, rank.to(tl.int64))
|
||||
tl.store(
|
||||
send_logprobs_ptr + dst * send_logprobs_stride + cols,
|
||||
logprobs.to(tl.float32),
|
||||
mask=dst_mask,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit(
|
||||
do_not_specialize=[
|
||||
"max_num_logits_per_req",
|
||||
"num_logprob_cols",
|
||||
"gathered_ids_stride",
|
||||
"gathered_logprobs_stride",
|
||||
]
|
||||
)
|
||||
def _unpack_logprobs_kernel(
|
||||
gathered_ids_ptr,
|
||||
gathered_ids_stride,
|
||||
gathered_logprobs_ptr,
|
||||
gathered_logprobs_stride,
|
||||
gathered_src_indices_ptr,
|
||||
cu_num_logits_ptr,
|
||||
logprob_token_ids_ptr,
|
||||
logprobs_ptr,
|
||||
selected_token_ranks_ptr,
|
||||
max_num_logits_per_req,
|
||||
num_logprob_cols,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
req_idx = tl.program_id(0)
|
||||
pos = tl.program_id(1)
|
||||
start = tl.load(cu_num_logits_ptr + req_idx)
|
||||
if pos < tl.load(cu_num_logits_ptr + req_idx + 1) - start:
|
||||
src = tl.load(gathered_src_indices_ptr + req_idx) * max_num_logits_per_req + pos
|
||||
dst = start + pos
|
||||
cols = tl.arange(0, BLOCK_SIZE)
|
||||
mask = cols < num_logprob_cols
|
||||
src_ids_row = gathered_ids_ptr + src * gathered_ids_stride
|
||||
ids = tl.load(src_ids_row + cols, mask=mask)
|
||||
tl.store(logprob_token_ids_ptr + dst * num_logprob_cols + cols, ids, mask=mask)
|
||||
rank = tl.load(src_ids_row + num_logprob_cols)
|
||||
tl.store(selected_token_ranks_ptr + dst, rank)
|
||||
logprobs = tl.load(
|
||||
gathered_logprobs_ptr + src * gathered_logprobs_stride + cols, mask=mask
|
||||
)
|
||||
tl.store(logprobs_ptr + dst * num_logprob_cols + cols, logprobs, mask=mask)
|
||||
|
||||
|
||||
def _gather_logprobs_tensors(
|
||||
local_output: SamplerOutput | None,
|
||||
metadata: BatchShardMetadata,
|
||||
global_batch: InputBatch,
|
||||
local_batch: InputBatch,
|
||||
logprobs_dims: tuple[int, int],
|
||||
device: torch.device,
|
||||
) -> LogprobsTensors:
|
||||
num_logprobs, max_token_ids = logprobs_dims
|
||||
num_logprob_cols = 1 + max(num_logprobs, max_token_ids)
|
||||
max_num_logits_per_req = metadata.max_num_logits_per_req
|
||||
num_send_rows = metadata.max_num_reqs_per_rank * max_num_logits_per_req
|
||||
block_size = triton.next_power_of_2(num_logprob_cols)
|
||||
|
||||
# Padded slots are never read back: gathered_src_indices only addresses
|
||||
# rows that an owner rank wrote.
|
||||
send_ids_ranks = torch.empty(
|
||||
num_send_rows, num_logprob_cols + 1, dtype=torch.int64, device=device
|
||||
)
|
||||
send_logprobs = torch.empty(
|
||||
num_send_rows, num_logprob_cols, dtype=torch.float32, device=device
|
||||
)
|
||||
if local_output is not None and local_output.logprobs_tensors is not None:
|
||||
lp = local_output.logprobs_tensors
|
||||
_pack_logprobs_kernel[(metadata.num_local_reqs, max_num_logits_per_req)](
|
||||
send_ids_ranks,
|
||||
send_ids_ranks.stride(0),
|
||||
send_logprobs,
|
||||
send_logprobs.stride(0),
|
||||
lp.logprob_token_ids,
|
||||
lp.logprob_token_ids.stride(0),
|
||||
lp.logprobs,
|
||||
lp.logprobs.stride(0),
|
||||
lp.selected_token_ranks,
|
||||
local_batch.cu_num_logits,
|
||||
max_num_logits_per_req,
|
||||
num_logprob_cols,
|
||||
lp.logprob_token_ids.shape[1],
|
||||
BLOCK_SIZE=block_size,
|
||||
)
|
||||
|
||||
gathered_ids_ranks = tensor_model_parallel_all_gather(send_ids_ranks, dim=0)
|
||||
gathered_logprobs = tensor_model_parallel_all_gather(send_logprobs, dim=0)
|
||||
|
||||
num_reqs = global_batch.num_reqs
|
||||
num_logits = int(global_batch.cu_num_logits_np[-1])
|
||||
logprob_token_ids = torch.empty(
|
||||
num_logits, num_logprob_cols, dtype=torch.int64, device=device
|
||||
)
|
||||
logprobs = torch.empty(
|
||||
num_logits, num_logprob_cols, dtype=torch.float32, device=device
|
||||
)
|
||||
selected_token_ranks = torch.empty(num_logits, dtype=torch.int64, device=device)
|
||||
_unpack_logprobs_kernel[(num_reqs, max_num_logits_per_req)](
|
||||
gathered_ids_ranks,
|
||||
gathered_ids_ranks.stride(0),
|
||||
gathered_logprobs,
|
||||
gathered_logprobs.stride(0),
|
||||
metadata.gathered_src_indices,
|
||||
global_batch.cu_num_logits,
|
||||
logprob_token_ids,
|
||||
logprobs,
|
||||
selected_token_ranks,
|
||||
max_num_logits_per_req,
|
||||
num_logprob_cols,
|
||||
BLOCK_SIZE=block_size,
|
||||
)
|
||||
return LogprobsTensors(
|
||||
logprob_token_ids=logprob_token_ids,
|
||||
logprobs=logprobs,
|
||||
selected_token_ranks=selected_token_ranks,
|
||||
cu_num_generated_tokens=(
|
||||
global_batch.cu_num_logits_np.tolist() if num_logits != num_reqs else None
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def gather_sampler_output(
|
||||
local_output: SamplerOutput | None,
|
||||
metadata: BatchShardMetadata,
|
||||
device: torch.device,
|
||||
global_batch: InputBatch,
|
||||
local_batch: InputBatch,
|
||||
gather_num_nans: bool = False,
|
||||
logprobs_dims: tuple[int, int] | None = None,
|
||||
) -> SamplerOutput:
|
||||
max_num_logits_per_req = metadata.max_num_logits_per_req
|
||||
num_packed_cols = max_num_logits_per_req + 2 + (1 if gather_num_nans else 0)
|
||||
block_size = triton.next_power_of_2(max_num_logits_per_req)
|
||||
|
||||
# Pack the sampler output tensors (excluding logprobs) into a single
|
||||
# tensor for the all-gather.
|
||||
packed = torch.empty(
|
||||
metadata.max_num_reqs_per_rank,
|
||||
num_packed_cols,
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
)
|
||||
if local_output is not None:
|
||||
assert local_output.num_sampled is not None
|
||||
assert local_output.num_rejected is not None
|
||||
assert not gather_num_nans or local_output.num_nans is not None
|
||||
num_src_cols = min(
|
||||
local_output.sampled_token_ids.shape[1], max_num_logits_per_req
|
||||
)
|
||||
_pack_sampler_output_kernel[(metadata.num_local_reqs,)](
|
||||
packed,
|
||||
packed.stride(0),
|
||||
local_output.sampled_token_ids,
|
||||
local_output.sampled_token_ids.stride(0),
|
||||
local_output.num_sampled,
|
||||
local_output.num_rejected,
|
||||
local_output.num_nans if gather_num_nans else None,
|
||||
local_batch.cu_num_logits if gather_num_nans else None,
|
||||
max_num_logits_per_req,
|
||||
num_src_cols,
|
||||
BLOCK_SIZE=block_size,
|
||||
)
|
||||
|
||||
# All-gather the sampler output tensors (excluding logprobs).
|
||||
gathered = tensor_model_parallel_all_gather(packed, dim=0)
|
||||
# All-gather the logprobs tensors.
|
||||
logprobs_tensors = None
|
||||
if logprobs_dims is not None:
|
||||
logprobs_tensors = _gather_logprobs_tensors(
|
||||
local_output, metadata, global_batch, local_batch, logprobs_dims, device
|
||||
)
|
||||
|
||||
# Unpack the gathered tensor into a sampler output object.
|
||||
num_reqs = metadata.gathered_src_indices.shape[0]
|
||||
sampled_token_ids = torch.empty(
|
||||
num_reqs, max_num_logits_per_req, dtype=torch.int64, device=device
|
||||
)
|
||||
num_sampled = torch.empty(num_reqs, dtype=torch.int32, device=device)
|
||||
num_rejected = torch.empty(num_reqs, dtype=torch.int32, device=device)
|
||||
num_nans = (
|
||||
torch.empty(num_reqs, dtype=torch.int32, device=device)
|
||||
if gather_num_nans
|
||||
else None
|
||||
)
|
||||
_unpack_gathered_output_kernel[(num_reqs,)](
|
||||
gathered,
|
||||
gathered.stride(0),
|
||||
metadata.gathered_src_indices,
|
||||
sampled_token_ids,
|
||||
num_sampled,
|
||||
num_rejected,
|
||||
num_nans,
|
||||
max_num_logits_per_req,
|
||||
BLOCK_SIZE=block_size,
|
||||
)
|
||||
return SamplerOutput(
|
||||
sampled_token_ids=sampled_token_ids,
|
||||
logprobs_tensors=logprobs_tensors,
|
||||
num_nans=num_nans,
|
||||
num_sampled=num_sampled,
|
||||
num_rejected=num_rejected,
|
||||
)
|
||||
@@ -103,6 +103,22 @@ class Sampler:
|
||||
if self.trace_replay_state is not None:
|
||||
self.trace_replay_state.apply_staged_writes()
|
||||
|
||||
def get_logprobs_dims(
|
||||
self, idx_mapping_np: np.ndarray, include_token_ids: bool = True
|
||||
) -> tuple[int, int] | None:
|
||||
"""(num_logprobs, max_per_req_token_ids) for the given requests, or
|
||||
None when none of them want logprobs."""
|
||||
max_num_logprobs = self.sampling_states.max_num_logprobs(idx_mapping_np)
|
||||
max_token_ids = (
|
||||
self.logprob_token_ids_state.max_num_token_ids(idx_mapping_np)
|
||||
if include_token_ids
|
||||
else 0
|
||||
)
|
||||
if max_num_logprobs == NO_LOGPROBS and max_token_ids == 0:
|
||||
return None
|
||||
num_logprobs = max_num_logprobs if max_num_logprobs != NO_LOGPROBS else 0
|
||||
return num_logprobs, max_token_ids
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
@@ -120,11 +136,7 @@ class Sampler:
|
||||
# that num_nans is computed before applying penalties and temperature.
|
||||
num_nans = get_num_nans(logits) if self.compute_nans else None
|
||||
|
||||
max_num_logprobs = self.sampling_states.max_num_logprobs(idx_mapping_np)
|
||||
max_per_req_token_ids = self.logprob_token_ids_state.max_num_token_ids(
|
||||
idx_mapping_np
|
||||
)
|
||||
return_logprobs = max_num_logprobs != NO_LOGPROBS or max_per_req_token_ids > 0
|
||||
logprobs_dims = self.get_logprobs_dims(idx_mapping_np)
|
||||
|
||||
sampled, processed_logits = self.sample(
|
||||
logits,
|
||||
@@ -134,7 +146,7 @@ class Sampler:
|
||||
pos,
|
||||
input_ids,
|
||||
expanded_local_pos,
|
||||
return_logprobs=return_logprobs,
|
||||
return_logprobs=logprobs_dims is not None,
|
||||
)
|
||||
|
||||
if self.trace_replay_state is not None:
|
||||
@@ -142,12 +154,12 @@ class Sampler:
|
||||
# computed logprobs reflect the real distribution of the forced token.
|
||||
self.trace_replay_state.apply_trace(sampled, idx_mapping)
|
||||
|
||||
if return_logprobs:
|
||||
if logprobs_dims is not None:
|
||||
num_logprobs, max_per_req_token_ids = logprobs_dims
|
||||
if self.logprobs_mode in PROCESSED_LOGPROBS_MODES:
|
||||
logits = processed_logits
|
||||
expanded_logits = logits.shape[0] != idx_mapping_np.shape[0]
|
||||
cu_num_logits = cu_num_logits_np.tolist() if expanded_logits else None
|
||||
num_logprobs = max_num_logprobs if max_num_logprobs != NO_LOGPROBS else 0
|
||||
logprobs_tensors = compute_topk_scores(
|
||||
logits,
|
||||
num_logprobs,
|
||||
|
||||
Reference in New Issue
Block a user