[Feature][Spec Decode] Support sampling mask replay for MRV2 MTP (#59359)

Signed-off-by: Cuiping Cheng <chengcuipingswu@163.com>
Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Cuiping Cheng <chengcuipingswu@163.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
vx120
2026-10-01 13:51:39 +08:00
committed by GitHub
co-authored by Cuiping Cheng aoshen02 Claude Opus 5.5
parent 37cecc8141
commit 70be6aae19
13 changed files with 583 additions and 68 deletions
+4 -1
View File
@@ -63,9 +63,12 @@ The mask is also available via the `/inference/v1/generate` HTTP endpoint:
| `top_k > 0` | Bounds mask size; pure top-p can produce vocab-sized masks |
| Model Runner V2 | Required by the async D2H copy pipeline |
Speculative decoding is supported; each emitted token gets the support of the
target distribution at its position.
The engine rejects unsupported combinations at startup or request time:
- Speculative decoding
- Speculative decoding with adaptive verification
- Diffusion models
- Custom logits processors (engine-level `--logits-processors`)
+91
View File
@@ -147,6 +147,97 @@ def test_rocm_mm_prefix_lm_disables_chunked_mm_input(
assert config.scheduler_config.disable_chunked_mm_input is expected
def _sampling_replay_config(
*,
return_sampling_mask: bool = True,
use_v2_model_runner: bool = True,
speculative_method: str | None = None,
rejection_sample_method: str = "standard",
adaptive: bool = False,
is_diffusion: bool = False,
logits_processors: list[str] | None = None,
logprobs_mode: str = "processed_logprobs",
):
speculative_config = None
if speculative_method is not None:
speculative_config = SimpleNamespace(
method=speculative_method,
enable_adaptive_verification=adaptive,
rejection_sample_method=rejection_sample_method,
)
return SimpleNamespace(
model_config=SimpleNamespace(
return_sampling_mask=return_sampling_mask,
is_diffusion=is_diffusion,
logits_processors=logits_processors or [],
logprobs_mode=logprobs_mode,
),
use_v2_model_runner=use_v2_model_runner,
speculative_config=speculative_config,
)
@pytest.mark.parametrize(
("config", "message"),
[
(_sampling_replay_config(), None),
*(
(_sampling_replay_config(speculative_method=method), None)
for method in (
"mtp",
"eagle",
"eagle3",
"dflash",
"dspark",
"draft_model",
)
),
*(
(
_sampling_replay_config(
speculative_method="mtp",
rejection_sample_method=rejection_sample_method,
),
None,
)
for rejection_sample_method in ("standard", "block", "synthetic")
),
(
_sampling_replay_config(
return_sampling_mask=False, speculative_method="dspark", adaptive=True
),
None,
),
(
_sampling_replay_config(speculative_method="dspark", adaptive=True),
"requires fixed verification boundaries",
),
(
_sampling_replay_config(is_diffusion=True),
"does not support diffusion models",
),
(
_sampling_replay_config(logits_processors=["custom"]),
"does not support custom logits processors",
),
(
_sampling_replay_config(logprobs_mode="raw_logprobs"),
"requires logprobs_mode='processed_logprobs'",
),
(
_sampling_replay_config(use_v2_model_runner=False),
"requires Model Runner V2",
),
],
)
def test_sampling_replay_config(config, message):
if message is None:
VllmConfig._verify_sampling_replay_config(config)
else:
with pytest.raises(ValueError, match=message):
VllmConfig._verify_sampling_replay_config(config)
def test_kda_recoverssm_derivation_is_revalidated():
config = SimpleNamespace(
cache_config=SimpleNamespace(
+43
View File
@@ -908,6 +908,49 @@ def test_update_from_output_routes_sampling_masks_by_request():
assert all(out.new_sampling_mask.offsets is None for out in outputs)
def test_update_from_output_routes_multi_position_sampling_masks():
scheduler = create_scheduler()
scheduler.return_sampling_mask = True
requests = create_requests(num_requests=2, max_tokens=10)
for req in requests:
req.num_computed_tokens = req.num_tokens
scheduler.requests[req.request_id] = req
scheduler.running.append(req)
req.status = RequestStatus.RUNNING
scheduler_output = SchedulerOutput(
scheduled_new_reqs=[],
scheduled_cached_reqs=CachedRequestData.make_empty(),
num_scheduled_tokens={req.request_id: 1 for req in requests},
total_num_scheduled_tokens=2,
scheduled_encoder_inputs={},
scheduled_spec_decode_tokens={},
num_common_prefix_blocks=[],
finished_req_ids=set(),
free_encoder_mm_hashes=[],
)
model_output = ModelRunnerOutput(
req_ids=[req.request_id for req in requests],
req_id_to_index={req.request_id: i for i, req in enumerate(requests)},
sampled_token_ids=[[1, 2], [3, 4, 5]],
logprobs=None,
prompt_logprobs_dict={},
pooler_output=[],
sampling_masks=SamplingMaskLists(
token_ids=np.array([1, 6, 2, 3, 7, 8, 4, 5, 9], dtype=np.int32),
offsets=np.array([0, 2, 3, 6, 7, 9]),
cu_num_generated_tokens=[0, 2, 5],
),
)
outputs = scheduler.update_from_output(scheduler_output, model_output)[0].outputs
assert [out.new_sampling_mask.to_nested_list() for out in outputs] == [
[[1, 6], [2]],
[[3, 7, 8], [4], [5, 9]],
]
def test_stop_via_update_from_output():
"""Test stopping behavior through update_from_output."""
scheduler = create_scheduler(num_speculative_tokens=1)
+26
View File
@@ -35,6 +35,7 @@ from vllm.v1.engine.output_processor import (
RequestState,
)
from vllm.v1.metrics.stats import IterationStats, PrefillStats, SchedulerStats
from vllm.v1.outputs import SamplingMaskLists
@pytest.mark.parametrize("flat_logprobs", [False, True])
@@ -62,6 +63,31 @@ def test_delta_output_without_new_tokens_returns_empty_logprobs(
assert len(output.logprobs) == 0
def test_completion_output_preserves_each_sampling_mask_position() -> None:
state = RequestState.__new__(RequestState)
state.detokenizer = MagicMock()
state.detokenizer.get_next_output_text.return_value = ""
state.logprobs_processor = MagicMock()
state.logprobs_processor.logprobs = None
state.logprobs_processor.cumulative_logprob = None
state.output_kind = RequestOutputKind.DELTA
state.request_index = 0
state.sampling_mask_chunks = [
SamplingMaskLists(
token_ids=np.array([10, 11, 20]),
offsets=np.array([0, 2, 3]),
),
SamplingMaskLists(token_ids=np.array([30, 31, 32])),
]
state.routed_experts_chunks = []
state.spec_decode_metrics = None
output = state._new_completion_output([1, 2, 3], FinishReason.LENGTH, None)
assert output.sampling_mask is not None
assert output.sampling_mask.token_ids == [[10, 11], [20], [30, 31, 32]]
def _ref_convert_id_to_token(
tokenizer: TokenizerLike,
token_id: int,
+67 -3
View File
@@ -13,6 +13,7 @@ from vllm.v1.outputs import (
LogprobsLists,
LogprobsTensors,
ModelRunnerOutput,
SamplingMaskLists,
)
from vllm.v1.sample.ops.topk_topp_sampler import apply_top_k_top_p
from vllm.v1.worker.gpu.sample.output import SamplingMaskTensors
@@ -63,8 +64,6 @@ def test_logprobs_tensors_tolists_with_tensor_boundaries():
def test_sampling_mask_lists_to_nested_list():
from vllm.v1.outputs import SamplingMaskLists
mask = SamplingMaskLists(
token_ids=np.array([10, 11, 12, 20, 21]),
offsets=np.array([0, 3, 5]),
@@ -76,6 +75,22 @@ def test_sampling_mask_lists_to_nested_list():
assert SamplingMaskLists(np.array([7, 9])).to_nested_list() == [[7, 9]]
def test_sampling_mask_lists_slices_multiple_positions_by_request():
masks = SamplingMaskLists(
token_ids=np.array([10, 11, 20, 30, 31, 32, 40]),
offsets=np.array([0, 2, 3, 6, 7]),
cu_num_generated_tokens=[0, 1, 3, 4],
)
single = masks.slice_request(0, 1)
multi = masks.slice_request(1, 2)
assert single.to_nested_list() == [[10, 11]]
assert single.offsets is None
assert multi.to_nested_list() == [[20], [30, 31, 32]]
assert multi.cu_num_generated_tokens is None
@pytest.mark.parametrize("max_num_kept", [512, 20_001])
@pytest.mark.skipif(
current_platform.is_xpu(),
@@ -102,7 +117,10 @@ def test_sampling_mask_tensors_match_finite_support(max_num_kept):
expected[1] = expected[7] = []
tensors = SamplingMaskTensors.from_logits(
logits.to(DEVICE_TYPE), num_sampled_tokens.to(DEVICE_TYPE), max_num_kept
logits.to(DEVICE_TYPE),
torch.arange(len(sizes) + 1, device=DEVICE_TYPE, dtype=torch.int32),
num_sampled_tokens.to(DEVICE_TYPE),
max_num_kept,
)
assert tensors.token_ids.shape[1] == min(max_num_kept, MAX_COMPACT_SUPPORT)
@@ -127,6 +145,7 @@ def test_sampling_mask_matches_processed_top_k_top_p_support():
tensors = SamplingMaskTensors.from_logits(
processed_logits,
cu_num_logits=torch.tensor([0, 1], dtype=torch.int32, device=DEVICE_TYPE),
num_sampled_tokens=torch.tensor([1], device=DEVICE_TYPE),
max_num_kept=3,
)
@@ -135,6 +154,50 @@ def test_sampling_mask_matches_processed_top_k_top_p_support():
assert result.to_nested_list() == [expected_token_ids]
def test_sampling_mask_tensors_uses_request_boundaries():
logits = torch.full((6, 8), -float("inf"), device=DEVICE_TYPE)
logits[0, [1, 3]] = 0
logits[2, [2, 4, 6]] = 0
logits[5, [0, 7]] = 0
counts = torch.tensor([1, 0, 1], device=DEVICE_TYPE, dtype=torch.int32)
tensors = SamplingMaskTensors.from_logits(
logits,
torch.tensor([0, 2, 5, 6], device=DEVICE_TYPE, dtype=torch.int32),
counts,
max_num_kept=3,
)
assert tensors.tolists().to_nested_list() == [[1, 3], [], [0, 7]]
def test_sampling_mask_tensors_multirow_request_layout():
rows_per_request = 4
counts = np.array([1, 2, 4, 0], dtype=np.int32)
logits = torch.full((16, 16), -float("inf"), device=DEVICE_TYPE)
expected = []
for row in [0, 4, 5, 8, 9, 10, 11]:
kept = torch.tensor(
[row % 16, (row + 3) % 16, (row + 7) % 16], device=DEVICE_TYPE
).unique(sorted=True)
logits[row, kept] = 0
expected.append(kept.tolist())
tensors = SamplingMaskTensors.from_logits(
logits,
torch.arange(0, 17, 4, device=DEVICE_TYPE, dtype=torch.int32),
torch.from_numpy(counts).to(DEVICE_TYPE),
max_num_kept=2,
rows_per_request=rows_per_request,
)
result = tensors.to_cpu_nonblocking().tolists(counts)
assert tensors.token_ids.shape == (16, 2)
assert tensors.rows_per_request == rows_per_request
assert result.to_nested_list() == expected
assert result.cu_num_generated_tokens == [0, 1, 3, 7, 7]
def test_sampling_mask_preserves_top_k_boundary_ties():
"""When the kept support is wider than `max_num_kept` (e.g. a top-k
boundary tie keeps more than k logits), the mask must fall back to the
@@ -147,6 +210,7 @@ def test_sampling_mask_preserves_top_k_boundary_ties():
tensors = SamplingMaskTensors.from_logits(
processed_logits,
cu_num_logits=torch.tensor([0, 1], dtype=torch.int32, device=DEVICE_TYPE),
num_sampled_tokens=torch.tensor([1], device=DEVICE_TYPE),
max_num_kept=3,
)
@@ -10,12 +10,53 @@ import torch
from vllm.config.model import PROCESSED_LOGPROBS_MODES, LogprobsMode
from vllm.platforms import current_platform
from vllm.v1.worker.gpu.sample.output import SamplingMaskTensors
from vllm.v1.worker.gpu.spec_decode.rejection_sampler import (
RejectionSampler,
_iter_request_chunks,
)
def _make_input_batch(
cu_num_logits_np: np.ndarray,
idx_mapping_np: np.ndarray,
device: torch.device,
) -> SimpleNamespace:
"""Minimal InputBatch fields read by RejectionSampler._verify_in_chunks."""
num_reqs = len(idx_mapping_np)
num_logits_per_req = np.diff(cu_num_logits_np)
return SimpleNamespace(
num_reqs=num_reqs,
cu_num_logits_np=cu_num_logits_np,
cu_num_logits=torch.from_numpy(cu_num_logits_np).to(device),
idx_mapping_np=idx_mapping_np,
idx_mapping=torch.from_numpy(idx_mapping_np).to(device),
expanded_idx_mapping=torch.from_numpy(
np.repeat(idx_mapping_np, num_logits_per_req)
).to(device),
expanded_local_pos=torch.from_numpy(
np.concatenate(
[np.arange(count, dtype=np.int32) for count in num_logits_per_req]
)
).to(device),
seq_lens_cpu_upper_bound=torch.full((num_reqs,), 64, dtype=torch.int32),
)
def _make_rejection_sampler(
sampler: SimpleNamespace, num_speculative_steps: int
) -> RejectionSampler:
"""RejectionSampler for standard, fixed-boundary, unwatermarked verification."""
rejection_sampler = object.__new__(RejectionSampler)
rejection_sampler.sampler = sampler
rejection_sampler.num_speculative_steps = num_speculative_steps
rejection_sampler.enable_adaptive_verification = False
rejection_sampler.synthetic_conditional_rates = None
rejection_sampler.use_block_verification = False
rejection_sampler.watermark_key = None
return rejection_sampler
def test_iter_request_chunks_preserves_request_boundaries():
cu_num_logits = np.array([0, 3, 4, 11, 13], dtype=np.int32)
@@ -33,28 +74,11 @@ def test_chunked_scores_match_full_batch(logprobs_mode: str):
cu_num_logits_np = np.array([0, 3, 4, 8, 10], dtype=np.int32)
num_logits_per_req = np.diff(cu_num_logits_np)
idx_mapping_np = np.array([7, 2, 9, 1], dtype=np.int32)
input_batch = SimpleNamespace(
num_reqs=4,
cu_num_logits_np=cu_num_logits_np,
cu_num_logits=torch.from_numpy(cu_num_logits_np).to(device),
idx_mapping_np=idx_mapping_np,
idx_mapping=torch.from_numpy(idx_mapping_np).to(device),
expanded_idx_mapping=torch.from_numpy(
np.repeat(idx_mapping_np, num_logits_per_req)
).to(device),
expanded_local_pos=torch.from_numpy(
np.concatenate(
[np.arange(count, dtype=np.int32) for count in num_logits_per_req]
)
).to(device),
seq_lens_cpu_upper_bound=torch.from_numpy(
np.array([10, 20, 30, 40], dtype=np.int32)
),
input_batch = _make_input_batch(cu_num_logits_np, idx_mapping_np, device)
rejection_sampler = _make_rejection_sampler(
SimpleNamespace(logprobs_mode=logprobs_mode, return_sampling_mask=False),
num_speculative_steps=3,
)
rejection_sampler = object.__new__(RejectionSampler)
rejection_sampler.sampler = SimpleNamespace(logprobs_mode=logprobs_mode)
rejection_sampler.num_speculative_steps = 3
rejection_sampler.enable_adaptive_verification = False
def fake_verify(
self,
@@ -75,14 +99,16 @@ def test_chunked_scores_match_full_batch(logprobs_mode: str):
rejection_sampler._verify = MethodType(fake_verify, rejection_sampler)
logits = torch.arange(170, dtype=torch.float32, device=device).view(10, 17)
sampled, num_sampled, chunked_logprobs = rejection_sampler._verify_in_chunks(
logits,
input_batch,
draft_logits=None,
draft_sampled=torch.arange(10, device=device),
pos=torch.arange(10, device=device),
max_chunk_logits=5,
max_num_logprobs=2,
sampled, num_sampled, chunked_logprobs, sampling_mask_tensors = (
rejection_sampler._verify_in_chunks(
logits,
input_batch,
draft_logits=None,
draft_sampled=torch.arange(10, device=device),
pos=torch.arange(10, device=device),
max_chunk_logits=5,
max_num_logprobs=2,
)
)
score_logits = logits + 1 if logprobs_mode in PROCESSED_LOGPROBS_MODES else logits
full_logprobs = rejection_sampler._get_logprobs_tensors(
@@ -111,3 +137,155 @@ def test_chunked_scores_match_full_batch(logprobs_mode: str):
chunked_logprobs.cu_num_generated_tokens
== full_logprobs.cu_num_generated_tokens
)
assert sampling_mask_tensors is None
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
def test_replay_on_off_preserves_rejection_sampling_and_rng(monkeypatch):
device = torch.device("cuda")
num_reqs = 3
num_speculative_steps = 3
rows_per_request = num_speculative_steps + 1
vocab_size = 8
cu_num_logits_np = np.arange(0, 13, rows_per_request, dtype=np.int32)
idx_mapping_np = np.arange(num_reqs, dtype=np.int32)
input_batch = _make_input_batch(cu_num_logits_np, idx_mapping_np, device)
processed_logits = torch.zeros((12, vocab_size), device=device)
processed_logits[0, 0] = -float("inf")
processed_logits[rows_per_request + 1, 1] = -float("inf")
draft_logits = torch.zeros(
(num_reqs, num_speculative_steps, vocab_size), device=device
)
draft_sampled = torch.zeros(
(num_reqs, rows_per_request), dtype=torch.int64, device=device
)
draft_sampled[:, 1:] = torch.tensor([0, 1, 2], device=device)
draft_sampled = draft_sampled.flatten()
sampler = SimpleNamespace(
logprobs_mode="processed_logprobs",
return_sampling_mask=False,
sampling_states=SimpleNamespace(
top_k=SimpleNamespace(np=np.full(num_reqs, vocab_size)),
temperature=SimpleNamespace(gpu=torch.ones(num_reqs, device=device)),
seeds=SimpleNamespace(
gpu=torch.arange(num_reqs, dtype=torch.int64, device=device)
),
),
use_fp64_gumbel=False,
apply_sampling_params=lambda logits, *_: logits,
)
rejection_sampler = _make_rejection_sampler(sampler, num_speculative_steps)
pos = torch.arange(12, dtype=torch.int32, device=device)
pack_calls: list[tuple[list[int], int, int]] = []
pack = SamplingMaskTensors.from_logits.__func__
def track_pack(
cls,
logits,
cu_num_logits,
num_sampled_tokens,
max_num_kept,
rows_per_request=1,
):
pack_calls.append(
(cu_num_logits.cpu().tolist(), max_num_kept, rows_per_request)
)
return pack(
cls,
logits,
cu_num_logits,
num_sampled_tokens,
max_num_kept,
rows_per_request,
)
monkeypatch.setattr(
SamplingMaskTensors,
"from_logits",
classmethod(track_pack),
)
rng_before = torch.cuda.get_rng_state()
off_sampled, off_counts, off_logprobs, off_masks = (
rejection_sampler._verify_in_chunks(
processed_logits,
input_batch,
draft_logits,
draft_sampled,
pos,
max_chunk_logits=8,
max_num_logprobs=0,
)
)
rng_after_off = torch.cuda.get_rng_state()
assert pack_calls == []
sampler.return_sampling_mask = True
on_sampled, on_counts, on_logprobs, on_masks = rejection_sampler._verify_in_chunks(
processed_logits,
input_batch,
draft_logits,
draft_sampled,
pos,
max_chunk_logits=8,
max_num_logprobs=0,
)
rng_after_on = torch.cuda.get_rng_state()
assert pack_calls == [
([0, 4, 8], vocab_size, rows_per_request),
([0, 4], vocab_size, rows_per_request),
]
assert off_masks is None and on_masks is not None
assert on_masks.rows_per_request == rows_per_request
assert on_masks.token_ids.shape == (num_reqs * rows_per_request, vocab_size)
assert on_masks.packed_mask.shape[0] == num_reqs * rows_per_request
assert off_counts.tolist() == [1, 2, 4]
assert torch.equal(off_counts, on_counts)
counts = off_counts.cpu().tolist()
for req_idx, count in enumerate(counts):
assert torch.equal(off_sampled[req_idx, :count], on_sampled[req_idx, :count])
mapped_rows = [
int(input_batch.cu_num_logits_np[req_idx]) + slot_idx
for req_idx, count in enumerate(counts)
for slot_idx in range(count)
]
assert off_logprobs is not None and on_logprobs is not None
assert torch.equal(
off_logprobs.logprob_token_ids[mapped_rows],
on_logprobs.logprob_token_ids[mapped_rows],
)
assert torch.equal(
off_logprobs.logprobs[mapped_rows], on_logprobs.logprobs[mapped_rows]
)
assert torch.equal(
off_logprobs.selected_token_ranks[mapped_rows],
on_logprobs.selected_token_ranks[mapped_rows],
)
assert torch.equal(rng_before, rng_after_off)
assert torch.equal(rng_after_off, rng_after_on)
masks = (
on_masks.to_cpu_nonblocking().tolists(on_counts.cpu().numpy()).to_nested_list()
)
expected_masks = [
torch.isfinite(processed_logits[row]).nonzero().flatten().tolist()
for row in mapped_rows
]
emitted = [
int(on_sampled[req_idx, slot_idx])
for req_idx, count in enumerate(on_counts.cpu().tolist())
for slot_idx in range(count)
]
expected_logprobs = torch.stack(
[
torch.log_softmax(processed_logits[row], dim=-1)[token_id]
for row, token_id in zip(mapped_rows, emitted)
]
)
assert masks == expected_masks
assert all(token_id in mask for token_id, mask in zip(emitted, masks))
assert torch.equal(on_logprobs.logprobs[mapped_rows, 0], expected_logprobs)
+8 -2
View File
@@ -1265,9 +1265,15 @@ class VllmConfig:
return
if not self.use_v2_model_runner:
raise ValueError("sampling distribution replay requires Model Runner V2")
if self.speculative_config is not None:
speculative_config = self.speculative_config
if (
speculative_config is not None
and speculative_config.enable_adaptive_verification
):
raise ValueError(
"sampling distribution replay does not support speculative decoding"
"sampling distribution replay with speculative decoding "
"requires fixed verification boundaries; disable adaptive "
"verification"
)
if model_config.is_diffusion:
raise ValueError(
+5 -1
View File
@@ -441,7 +441,11 @@ class RequestState:
sampling_mask = None
if finished and self.sampling_mask_chunks:
sampling_mask = SamplingMask(
[chunk.token_ids.tolist() for chunk in self.sampling_mask_chunks]
[
position
for chunk in self.sampling_mask_chunks
for position in chunk.to_nested_list()
]
)
# Concatenate routed experts on finish
+10 -4
View File
@@ -57,19 +57,25 @@ class LogprobsLists(NamedTuple):
class SamplingMaskLists(NamedTuple):
"""CSR sampling masks; a step slice holds one position (``offsets=None``)."""
"""CSR sampling masks; a request slice may contain multiple positions."""
# [num_kept_tokens]
token_ids: np.ndarray
# [num_positions + 1], or None for a single position
offsets: np.ndarray | None = None
# Unused with one position per request; kept for the wire layout.
# [num_requests + 1] for multi-position request batches.
cu_num_generated_tokens: list[int] | None = None
def slice_request(self, req_idx: int, num_positions: int) -> "SamplingMaskLists":
assert num_positions == 1 and self.offsets is not None
assert self.offsets is not None
cu = self.cu_num_generated_tokens
start = req_idx if cu is None else cu[req_idx]
end = start + num_positions
lo, hi = self.offsets[start], self.offsets[end]
if num_positions == 1:
return SamplingMaskLists(self.token_ids[lo:hi])
return SamplingMaskLists(
self.token_ids[self.offsets[req_idx] : self.offsets[req_idx + 1]]
self.token_ids[lo:hi], self.offsets[start : end + 1] - lo
)
def to_nested_list(self) -> list[list[int]]:
+1 -1
View File
@@ -185,7 +185,7 @@ class AsyncOutput(AsyncModelRunnerOutput):
if self.sampling_mask_tensors is not None:
self.model_runner_output.sampling_masks = (
self.sampling_mask_tensors.tolists()
self.sampling_mask_tensors.tolists(self.num_sampled_tokens_np)
)
if self.num_nans is not None:
+70 -21
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
from collections.abc import Sequence
from dataclasses import dataclass
from typing import NamedTuple
@@ -27,6 +28,7 @@ def _compact_sampling_mask_kernel(
logits_ptr,
logits_row_stride,
logits_col_stride,
cu_num_logits_ptr,
num_sampled_tokens_ptr,
token_ids_ptr,
token_ids_row_stride,
@@ -35,25 +37,32 @@ def _compact_sampling_mask_kernel(
counts_ptr,
vocab_size,
max_num_kept,
ROWS_PER_REQUEST: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
"""Per row: first ``max_num_kept`` finite-logit ids, the full count, the bitmask."""
req_idx = tl.program_id(0)
is_active = tl.load(num_sampled_tokens_ptr + req_idx) > 0
output_row = tl.program_id(0)
req_idx = output_row // ROWS_PER_REQUEST
slot_idx = output_row % ROWS_PER_REQUEST
if slot_idx >= tl.load(num_sampled_tokens_ptr + req_idx):
tl.store(counts_ptr + output_row, 0)
return
source_row = tl.load(cu_num_logits_ptr + req_idx) + slot_idx
count = tl.zeros((), dtype=tl.int32)
for start_idx in range(0, vocab_size, BLOCK_SIZE):
offsets = start_idx + tl.arange(0, BLOCK_SIZE)
logits = tl.load(
logits_ptr + req_idx * logits_row_stride + offsets * logits_col_stride,
logits_ptr + source_row * logits_row_stride + offsets * logits_col_stride,
mask=offsets < vocab_size,
other=-float("inf"),
)
keep = (logits > -float("inf")) & (logits < float("inf")) & is_active
keep = (logits > -float("inf")) & (logits < float("inf"))
keep_i32 = keep.to(tl.int32)
pos = count + tl.cumsum(keep_i32, axis=0) - keep_i32
tl.store(
token_ids_ptr + req_idx * token_ids_row_stride + pos,
token_ids_ptr + output_row * token_ids_row_stride + pos,
offsets.to(tl.int32),
mask=keep & (pos < max_num_kept),
)
@@ -62,12 +71,12 @@ def _compact_sampling_mask_kernel(
bits = tl.reshape(keep_i32, (BLOCK_SIZE // 8, 8)) << tl.arange(0, 8)[None, :]
byte_offsets = start_idx // 8 + tl.arange(0, BLOCK_SIZE // 8)
tl.store(
packed_mask_ptr + req_idx * packed_mask_row_stride + byte_offsets,
packed_mask_ptr + output_row * packed_mask_row_stride + byte_offsets,
tl.sum(bits, axis=1).to(tl.uint8),
mask=byte_offsets < tl.cdiv(vocab_size, 8),
)
tl.store(counts_ptr + req_idx, count)
tl.store(counts_ptr + output_row, count)
# Bounds the [num_reqs, width] int32 buffer; wider rows use the bitmask instead.
@@ -78,37 +87,46 @@ class SamplingMaskTensors(NamedTuple):
"""Device-side masks pending async D2H: compact ids, plus the bitmask as
the exact fallback for rows wider than ``max_num_kept``."""
# [num_requests, max_num_kept]
# [num_requests * rows_per_request, max_num_kept]
token_ids: torch.Tensor
# [num_requests, ceil(vocab_size / 8)]
# [num_requests * rows_per_request, ceil(vocab_size / 8)]
packed_mask: torch.Tensor
# [num_requests]
# [num_requests * rows_per_request]
counts: torch.Tensor
vocab_size: int
# Fixed request-major output slots. Ordinary sampling has one slot.
rows_per_request: int = 1
@classmethod
def from_logits(
cls,
logits: torch.Tensor,
cu_num_logits: torch.Tensor,
num_sampled_tokens: torch.Tensor,
max_num_kept: int,
rows_per_request: int = 1,
) -> SamplingMaskTensors:
"""Capture the finite-logit support of every row with a sampled token."""
num_reqs, vocab_size = logits.shape
"""Capture committed target supports in fixed request-major slots."""
num_reqs = num_sampled_tokens.shape[0]
vocab_size = logits.shape[1]
max_num_kept = min(max_num_kept, vocab_size, MAX_COMPACT_SUPPORT)
num_output_rows = num_reqs * rows_per_request
device = logits.device
token_ids = torch.empty(
(num_reqs, max_num_kept), dtype=torch.int32, device=device
(num_output_rows, max_num_kept), dtype=torch.int32, device=device
)
packed_mask = torch.empty(
(num_reqs, (vocab_size + 7) // 8), dtype=torch.uint8, device=device
(num_output_rows, (vocab_size + 7) // 8),
dtype=torch.uint8,
device=device,
)
counts = torch.empty(num_reqs, dtype=torch.int32, device=device)
_compact_sampling_mask_kernel[(num_reqs,)](
counts = torch.empty(num_output_rows, dtype=torch.int32, device=device)
_compact_sampling_mask_kernel[(num_output_rows,)](
logits,
logits.stride(0),
logits.stride(1),
cu_num_logits,
num_sampled_tokens,
token_ids,
token_ids.stride(0),
@@ -117,9 +135,23 @@ class SamplingMaskTensors(NamedTuple):
counts,
vocab_size,
max_num_kept,
ROWS_PER_REQUEST=rows_per_request,
BLOCK_SIZE=8192,
)
return cls(token_ids, packed_mask, counts, vocab_size)
return cls(token_ids, packed_mask, counts, vocab_size, rows_per_request)
@classmethod
def cat(cls, chunks: Sequence[SamplingMaskTensors]) -> SamplingMaskTensors:
"""Join request-contiguous chunks that share one output layout."""
first = chunks[0]
assert len({(c.vocab_size, c.rows_per_request) for c in chunks}) == 1
return cls(
torch.cat([chunk.token_ids for chunk in chunks]),
torch.cat([chunk.packed_mask for chunk in chunks]),
torch.cat([chunk.counts for chunk in chunks]),
first.vocab_size,
first.rows_per_request,
)
def to_cpu_nonblocking(self) -> SamplingMaskTensors:
if self.token_ids.device.type == "cpu":
@@ -129,15 +161,31 @@ class SamplingMaskTensors(NamedTuple):
self.packed_mask.to("cpu", non_blocking=True),
self.counts.to("cpu", non_blocking=True),
self.vocab_size,
self.rows_per_request,
)
def tolists(self) -> SamplingMaskLists:
"""CSR over all requests; rows without a sampled token are empty."""
def tolists(
self, num_sampled_tokens: np.ndarray | None = None
) -> SamplingMaskLists:
"""Convert fixed output rows to the scheduler's CSR representation."""
counts = self.counts.cpu().numpy()
token_ids = self.token_ids.cpu().numpy()
packed_mask = self.packed_mask.cpu().numpy()
width = token_ids.shape[1]
cu_num_generated_tokens = None
if self.rows_per_request == 1:
sampled_rows = np.arange(len(counts))
else:
assert num_sampled_tokens is not None
active_slots = (
np.arange(self.rows_per_request)[None, :] < num_sampled_tokens[:, None]
)
sampled_rows = np.flatnonzero(active_slots)
cu_num_generated_tokens = np.cumsum(
np.concatenate(([0], num_sampled_tokens))
).tolist()
def support(row: int) -> np.ndarray:
if counts[row] <= width:
return token_ids[row, : counts[row]]
@@ -147,7 +195,8 @@ class SamplingMaskTensors(NamedTuple):
)
return np.flatnonzero(bits).astype(np.int32, copy=False)
supports = [support(row) for row in range(len(counts))]
supports = [support(row) for row in sampled_rows]
offsets = np.zeros(len(supports) + 1, dtype=np.int64)
np.cumsum([len(s) for s in supports], out=offsets[1:])
return SamplingMaskLists(np.concatenate(supports), offsets)
flat = np.concatenate([np.empty(0, dtype=np.int32), *supports])
return SamplingMaskLists(flat, offsets, cu_num_generated_tokens)
+4 -1
View File
@@ -203,7 +203,10 @@ class Sampler:
# Size by the validated top_k batch max; wider supports use the bitmask.
max_num_kept = int(np.max(self.sampling_states.top_k.np[idx_mapping_np]))
sampling_mask_tensors = SamplingMaskTensors.from_logits(
processed_logits, num_sampled, max_num_kept
processed_logits,
input_batch.cu_num_logits,
num_sampled,
max_num_kept,
)
# These are GPU tensors.
@@ -18,7 +18,7 @@ from vllm.v1.worker.gpu.input_batch import (
)
from vllm.v1.worker.gpu.metrics.logits import get_num_nans
from vllm.v1.worker.gpu.sample.logprob import compute_topk_scores
from vllm.v1.worker.gpu.sample.output import SamplerOutput
from vllm.v1.worker.gpu.sample.output import SamplerOutput, SamplingMaskTensors
from vllm.v1.worker.gpu.sample.sampler import Sampler
from vllm.v1.worker.gpu.sample.states import NO_LOGPROBS
from vllm.v1.worker.gpu.spec_decode.rejection_sampler_utils import (
@@ -296,7 +296,12 @@ class RejectionSampler:
max_chunk_logits: int,
max_num_logprobs: int,
verify_draft_sampled: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, LogprobsTensors | None]:
) -> tuple[
torch.Tensor,
torch.Tensor,
LogprobsTensors | None,
SamplingMaskTensors | None,
]:
if verify_draft_sampled is None:
verify_draft_sampled = draft_sampled
cu_num_logits_np = input_batch.cu_num_logits_np
@@ -316,6 +321,14 @@ class RejectionSampler:
sampled_chunks: list[torch.Tensor] = []
num_sampled_chunks: list[torch.Tensor] = []
logprobs_chunks: list[LogprobsTensors] = []
sampling_mask_chunks: list[SamplingMaskTensors] = []
max_num_kept = None
if self.sampler.return_sampling_mask:
max_num_kept = int(
np.max(
self.sampler.sampling_states.top_k.np[input_batch.idx_mapping_np]
)
)
for start, end in request_chunks:
lo = int(cu_num_logits_np[start])
@@ -346,13 +359,31 @@ class RejectionSampler:
)
if chunk_logprobs is not None:
logprobs_chunks.append(chunk_logprobs)
if max_num_kept is not None:
sampling_mask_chunks.append(
SamplingMaskTensors.from_logits(
processed_logits,
chunk_cu_num_logits,
num_sampled,
max_num_kept,
self.num_speculative_steps + 1,
)
)
del processed_logits
sampled_chunks.append(sampled)
num_sampled_chunks.append(num_sampled)
if len(sampled_chunks) == 1:
logprobs_tensors = logprobs_chunks[0] if logprobs_chunks else None
return sampled_chunks[0], num_sampled_chunks[0], logprobs_tensors
sampling_mask_tensors = (
sampling_mask_chunks[0] if sampling_mask_chunks else None
)
return (
sampled_chunks[0],
num_sampled_chunks[0],
logprobs_tensors,
sampling_mask_tensors,
)
logprobs_tensors = None
if logprobs_chunks:
@@ -366,7 +397,12 @@ class RejectionSampler:
sampled = torch.cat(sampled_chunks)
num_sampled = torch.cat(num_sampled_chunks)
return sampled, num_sampled, logprobs_tensors
sampling_mask_tensors = (
SamplingMaskTensors.cat(sampling_mask_chunks)
if sampling_mask_chunks
else None
)
return sampled, num_sampled, logprobs_tensors, sampling_mask_tensors
def __call__(
self,
@@ -400,7 +436,12 @@ class RejectionSampler:
input_batch.idx_mapping_np
)
chunk_logit_limit = get_max_chunk_logits(logits.shape[1])
sampled, num_sampled, logprobs_tensors = self._verify_in_chunks(
(
sampled,
num_sampled,
logprobs_tensors,
sampling_mask_tensors,
) = self._verify_in_chunks(
logits,
input_batch,
draft_logits,
@@ -425,4 +466,5 @@ class RejectionSampler:
num_nans=num_nans,
num_sampled=num_sampled,
num_rejected=num_rejected,
sampling_mask_tensors=sampling_mask_tensors,
)