[Feature][Spec Decode] Support sampling mask replay for MRV2 MTP (#59359)
Signed-off-by: Cuiping Cheng <chengcuipingswu@163.com> Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Cuiping Cheng <chengcuipingswu@163.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Cuiping Cheng
aoshen02
Claude Opus 5.5
parent
37cecc8141
commit
70be6aae19
@@ -147,6 +147,97 @@ def test_rocm_mm_prefix_lm_disables_chunked_mm_input(
|
||||
assert config.scheduler_config.disable_chunked_mm_input is expected
|
||||
|
||||
|
||||
def _sampling_replay_config(
|
||||
*,
|
||||
return_sampling_mask: bool = True,
|
||||
use_v2_model_runner: bool = True,
|
||||
speculative_method: str | None = None,
|
||||
rejection_sample_method: str = "standard",
|
||||
adaptive: bool = False,
|
||||
is_diffusion: bool = False,
|
||||
logits_processors: list[str] | None = None,
|
||||
logprobs_mode: str = "processed_logprobs",
|
||||
):
|
||||
speculative_config = None
|
||||
if speculative_method is not None:
|
||||
speculative_config = SimpleNamespace(
|
||||
method=speculative_method,
|
||||
enable_adaptive_verification=adaptive,
|
||||
rejection_sample_method=rejection_sample_method,
|
||||
)
|
||||
return SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
return_sampling_mask=return_sampling_mask,
|
||||
is_diffusion=is_diffusion,
|
||||
logits_processors=logits_processors or [],
|
||||
logprobs_mode=logprobs_mode,
|
||||
),
|
||||
use_v2_model_runner=use_v2_model_runner,
|
||||
speculative_config=speculative_config,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("config", "message"),
|
||||
[
|
||||
(_sampling_replay_config(), None),
|
||||
*(
|
||||
(_sampling_replay_config(speculative_method=method), None)
|
||||
for method in (
|
||||
"mtp",
|
||||
"eagle",
|
||||
"eagle3",
|
||||
"dflash",
|
||||
"dspark",
|
||||
"draft_model",
|
||||
)
|
||||
),
|
||||
*(
|
||||
(
|
||||
_sampling_replay_config(
|
||||
speculative_method="mtp",
|
||||
rejection_sample_method=rejection_sample_method,
|
||||
),
|
||||
None,
|
||||
)
|
||||
for rejection_sample_method in ("standard", "block", "synthetic")
|
||||
),
|
||||
(
|
||||
_sampling_replay_config(
|
||||
return_sampling_mask=False, speculative_method="dspark", adaptive=True
|
||||
),
|
||||
None,
|
||||
),
|
||||
(
|
||||
_sampling_replay_config(speculative_method="dspark", adaptive=True),
|
||||
"requires fixed verification boundaries",
|
||||
),
|
||||
(
|
||||
_sampling_replay_config(is_diffusion=True),
|
||||
"does not support diffusion models",
|
||||
),
|
||||
(
|
||||
_sampling_replay_config(logits_processors=["custom"]),
|
||||
"does not support custom logits processors",
|
||||
),
|
||||
(
|
||||
_sampling_replay_config(logprobs_mode="raw_logprobs"),
|
||||
"requires logprobs_mode='processed_logprobs'",
|
||||
),
|
||||
(
|
||||
_sampling_replay_config(use_v2_model_runner=False),
|
||||
"requires Model Runner V2",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_sampling_replay_config(config, message):
|
||||
if message is None:
|
||||
VllmConfig._verify_sampling_replay_config(config)
|
||||
else:
|
||||
with pytest.raises(ValueError, match=message):
|
||||
VllmConfig._verify_sampling_replay_config(config)
|
||||
|
||||
|
||||
def test_kda_recoverssm_derivation_is_revalidated():
|
||||
config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(
|
||||
|
||||
@@ -908,6 +908,49 @@ def test_update_from_output_routes_sampling_masks_by_request():
|
||||
assert all(out.new_sampling_mask.offsets is None for out in outputs)
|
||||
|
||||
|
||||
def test_update_from_output_routes_multi_position_sampling_masks():
|
||||
scheduler = create_scheduler()
|
||||
scheduler.return_sampling_mask = True
|
||||
requests = create_requests(num_requests=2, max_tokens=10)
|
||||
for req in requests:
|
||||
req.num_computed_tokens = req.num_tokens
|
||||
scheduler.requests[req.request_id] = req
|
||||
scheduler.running.append(req)
|
||||
req.status = RequestStatus.RUNNING
|
||||
|
||||
scheduler_output = SchedulerOutput(
|
||||
scheduled_new_reqs=[],
|
||||
scheduled_cached_reqs=CachedRequestData.make_empty(),
|
||||
num_scheduled_tokens={req.request_id: 1 for req in requests},
|
||||
total_num_scheduled_tokens=2,
|
||||
scheduled_encoder_inputs={},
|
||||
scheduled_spec_decode_tokens={},
|
||||
num_common_prefix_blocks=[],
|
||||
finished_req_ids=set(),
|
||||
free_encoder_mm_hashes=[],
|
||||
)
|
||||
model_output = ModelRunnerOutput(
|
||||
req_ids=[req.request_id for req in requests],
|
||||
req_id_to_index={req.request_id: i for i, req in enumerate(requests)},
|
||||
sampled_token_ids=[[1, 2], [3, 4, 5]],
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
sampling_masks=SamplingMaskLists(
|
||||
token_ids=np.array([1, 6, 2, 3, 7, 8, 4, 5, 9], dtype=np.int32),
|
||||
offsets=np.array([0, 2, 3, 6, 7, 9]),
|
||||
cu_num_generated_tokens=[0, 2, 5],
|
||||
),
|
||||
)
|
||||
|
||||
outputs = scheduler.update_from_output(scheduler_output, model_output)[0].outputs
|
||||
|
||||
assert [out.new_sampling_mask.to_nested_list() for out in outputs] == [
|
||||
[[1, 6], [2]],
|
||||
[[3, 7, 8], [4], [5, 9]],
|
||||
]
|
||||
|
||||
|
||||
def test_stop_via_update_from_output():
|
||||
"""Test stopping behavior through update_from_output."""
|
||||
scheduler = create_scheduler(num_speculative_tokens=1)
|
||||
|
||||
@@ -35,6 +35,7 @@ from vllm.v1.engine.output_processor import (
|
||||
RequestState,
|
||||
)
|
||||
from vllm.v1.metrics.stats import IterationStats, PrefillStats, SchedulerStats
|
||||
from vllm.v1.outputs import SamplingMaskLists
|
||||
|
||||
|
||||
@pytest.mark.parametrize("flat_logprobs", [False, True])
|
||||
@@ -62,6 +63,31 @@ def test_delta_output_without_new_tokens_returns_empty_logprobs(
|
||||
assert len(output.logprobs) == 0
|
||||
|
||||
|
||||
def test_completion_output_preserves_each_sampling_mask_position() -> None:
|
||||
state = RequestState.__new__(RequestState)
|
||||
state.detokenizer = MagicMock()
|
||||
state.detokenizer.get_next_output_text.return_value = ""
|
||||
state.logprobs_processor = MagicMock()
|
||||
state.logprobs_processor.logprobs = None
|
||||
state.logprobs_processor.cumulative_logprob = None
|
||||
state.output_kind = RequestOutputKind.DELTA
|
||||
state.request_index = 0
|
||||
state.sampling_mask_chunks = [
|
||||
SamplingMaskLists(
|
||||
token_ids=np.array([10, 11, 20]),
|
||||
offsets=np.array([0, 2, 3]),
|
||||
),
|
||||
SamplingMaskLists(token_ids=np.array([30, 31, 32])),
|
||||
]
|
||||
state.routed_experts_chunks = []
|
||||
state.spec_decode_metrics = None
|
||||
|
||||
output = state._new_completion_output([1, 2, 3], FinishReason.LENGTH, None)
|
||||
|
||||
assert output.sampling_mask is not None
|
||||
assert output.sampling_mask.token_ids == [[10, 11], [20], [30, 31, 32]]
|
||||
|
||||
|
||||
def _ref_convert_id_to_token(
|
||||
tokenizer: TokenizerLike,
|
||||
token_id: int,
|
||||
|
||||
@@ -13,6 +13,7 @@ from vllm.v1.outputs import (
|
||||
LogprobsLists,
|
||||
LogprobsTensors,
|
||||
ModelRunnerOutput,
|
||||
SamplingMaskLists,
|
||||
)
|
||||
from vllm.v1.sample.ops.topk_topp_sampler import apply_top_k_top_p
|
||||
from vllm.v1.worker.gpu.sample.output import SamplingMaskTensors
|
||||
@@ -63,8 +64,6 @@ def test_logprobs_tensors_tolists_with_tensor_boundaries():
|
||||
|
||||
|
||||
def test_sampling_mask_lists_to_nested_list():
|
||||
from vllm.v1.outputs import SamplingMaskLists
|
||||
|
||||
mask = SamplingMaskLists(
|
||||
token_ids=np.array([10, 11, 12, 20, 21]),
|
||||
offsets=np.array([0, 3, 5]),
|
||||
@@ -76,6 +75,22 @@ def test_sampling_mask_lists_to_nested_list():
|
||||
assert SamplingMaskLists(np.array([7, 9])).to_nested_list() == [[7, 9]]
|
||||
|
||||
|
||||
def test_sampling_mask_lists_slices_multiple_positions_by_request():
|
||||
masks = SamplingMaskLists(
|
||||
token_ids=np.array([10, 11, 20, 30, 31, 32, 40]),
|
||||
offsets=np.array([0, 2, 3, 6, 7]),
|
||||
cu_num_generated_tokens=[0, 1, 3, 4],
|
||||
)
|
||||
|
||||
single = masks.slice_request(0, 1)
|
||||
multi = masks.slice_request(1, 2)
|
||||
|
||||
assert single.to_nested_list() == [[10, 11]]
|
||||
assert single.offsets is None
|
||||
assert multi.to_nested_list() == [[20], [30, 31, 32]]
|
||||
assert multi.cu_num_generated_tokens is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("max_num_kept", [512, 20_001])
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_xpu(),
|
||||
@@ -102,7 +117,10 @@ def test_sampling_mask_tensors_match_finite_support(max_num_kept):
|
||||
expected[1] = expected[7] = []
|
||||
|
||||
tensors = SamplingMaskTensors.from_logits(
|
||||
logits.to(DEVICE_TYPE), num_sampled_tokens.to(DEVICE_TYPE), max_num_kept
|
||||
logits.to(DEVICE_TYPE),
|
||||
torch.arange(len(sizes) + 1, device=DEVICE_TYPE, dtype=torch.int32),
|
||||
num_sampled_tokens.to(DEVICE_TYPE),
|
||||
max_num_kept,
|
||||
)
|
||||
|
||||
assert tensors.token_ids.shape[1] == min(max_num_kept, MAX_COMPACT_SUPPORT)
|
||||
@@ -127,6 +145,7 @@ def test_sampling_mask_matches_processed_top_k_top_p_support():
|
||||
|
||||
tensors = SamplingMaskTensors.from_logits(
|
||||
processed_logits,
|
||||
cu_num_logits=torch.tensor([0, 1], dtype=torch.int32, device=DEVICE_TYPE),
|
||||
num_sampled_tokens=torch.tensor([1], device=DEVICE_TYPE),
|
||||
max_num_kept=3,
|
||||
)
|
||||
@@ -135,6 +154,50 @@ def test_sampling_mask_matches_processed_top_k_top_p_support():
|
||||
assert result.to_nested_list() == [expected_token_ids]
|
||||
|
||||
|
||||
def test_sampling_mask_tensors_uses_request_boundaries():
|
||||
logits = torch.full((6, 8), -float("inf"), device=DEVICE_TYPE)
|
||||
logits[0, [1, 3]] = 0
|
||||
logits[2, [2, 4, 6]] = 0
|
||||
logits[5, [0, 7]] = 0
|
||||
counts = torch.tensor([1, 0, 1], device=DEVICE_TYPE, dtype=torch.int32)
|
||||
|
||||
tensors = SamplingMaskTensors.from_logits(
|
||||
logits,
|
||||
torch.tensor([0, 2, 5, 6], device=DEVICE_TYPE, dtype=torch.int32),
|
||||
counts,
|
||||
max_num_kept=3,
|
||||
)
|
||||
|
||||
assert tensors.tolists().to_nested_list() == [[1, 3], [], [0, 7]]
|
||||
|
||||
|
||||
def test_sampling_mask_tensors_multirow_request_layout():
|
||||
rows_per_request = 4
|
||||
counts = np.array([1, 2, 4, 0], dtype=np.int32)
|
||||
logits = torch.full((16, 16), -float("inf"), device=DEVICE_TYPE)
|
||||
expected = []
|
||||
for row in [0, 4, 5, 8, 9, 10, 11]:
|
||||
kept = torch.tensor(
|
||||
[row % 16, (row + 3) % 16, (row + 7) % 16], device=DEVICE_TYPE
|
||||
).unique(sorted=True)
|
||||
logits[row, kept] = 0
|
||||
expected.append(kept.tolist())
|
||||
|
||||
tensors = SamplingMaskTensors.from_logits(
|
||||
logits,
|
||||
torch.arange(0, 17, 4, device=DEVICE_TYPE, dtype=torch.int32),
|
||||
torch.from_numpy(counts).to(DEVICE_TYPE),
|
||||
max_num_kept=2,
|
||||
rows_per_request=rows_per_request,
|
||||
)
|
||||
result = tensors.to_cpu_nonblocking().tolists(counts)
|
||||
|
||||
assert tensors.token_ids.shape == (16, 2)
|
||||
assert tensors.rows_per_request == rows_per_request
|
||||
assert result.to_nested_list() == expected
|
||||
assert result.cu_num_generated_tokens == [0, 1, 3, 7, 7]
|
||||
|
||||
|
||||
def test_sampling_mask_preserves_top_k_boundary_ties():
|
||||
"""When the kept support is wider than `max_num_kept` (e.g. a top-k
|
||||
boundary tie keeps more than k logits), the mask must fall back to the
|
||||
@@ -147,6 +210,7 @@ def test_sampling_mask_preserves_top_k_boundary_ties():
|
||||
|
||||
tensors = SamplingMaskTensors.from_logits(
|
||||
processed_logits,
|
||||
cu_num_logits=torch.tensor([0, 1], dtype=torch.int32, device=DEVICE_TYPE),
|
||||
num_sampled_tokens=torch.tensor([1], device=DEVICE_TYPE),
|
||||
max_num_kept=3,
|
||||
)
|
||||
|
||||
@@ -10,12 +10,53 @@ import torch
|
||||
|
||||
from vllm.config.model import PROCESSED_LOGPROBS_MODES, LogprobsMode
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.worker.gpu.sample.output import SamplingMaskTensors
|
||||
from vllm.v1.worker.gpu.spec_decode.rejection_sampler import (
|
||||
RejectionSampler,
|
||||
_iter_request_chunks,
|
||||
)
|
||||
|
||||
|
||||
def _make_input_batch(
|
||||
cu_num_logits_np: np.ndarray,
|
||||
idx_mapping_np: np.ndarray,
|
||||
device: torch.device,
|
||||
) -> SimpleNamespace:
|
||||
"""Minimal InputBatch fields read by RejectionSampler._verify_in_chunks."""
|
||||
num_reqs = len(idx_mapping_np)
|
||||
num_logits_per_req = np.diff(cu_num_logits_np)
|
||||
return SimpleNamespace(
|
||||
num_reqs=num_reqs,
|
||||
cu_num_logits_np=cu_num_logits_np,
|
||||
cu_num_logits=torch.from_numpy(cu_num_logits_np).to(device),
|
||||
idx_mapping_np=idx_mapping_np,
|
||||
idx_mapping=torch.from_numpy(idx_mapping_np).to(device),
|
||||
expanded_idx_mapping=torch.from_numpy(
|
||||
np.repeat(idx_mapping_np, num_logits_per_req)
|
||||
).to(device),
|
||||
expanded_local_pos=torch.from_numpy(
|
||||
np.concatenate(
|
||||
[np.arange(count, dtype=np.int32) for count in num_logits_per_req]
|
||||
)
|
||||
).to(device),
|
||||
seq_lens_cpu_upper_bound=torch.full((num_reqs,), 64, dtype=torch.int32),
|
||||
)
|
||||
|
||||
|
||||
def _make_rejection_sampler(
|
||||
sampler: SimpleNamespace, num_speculative_steps: int
|
||||
) -> RejectionSampler:
|
||||
"""RejectionSampler for standard, fixed-boundary, unwatermarked verification."""
|
||||
rejection_sampler = object.__new__(RejectionSampler)
|
||||
rejection_sampler.sampler = sampler
|
||||
rejection_sampler.num_speculative_steps = num_speculative_steps
|
||||
rejection_sampler.enable_adaptive_verification = False
|
||||
rejection_sampler.synthetic_conditional_rates = None
|
||||
rejection_sampler.use_block_verification = False
|
||||
rejection_sampler.watermark_key = None
|
||||
return rejection_sampler
|
||||
|
||||
|
||||
def test_iter_request_chunks_preserves_request_boundaries():
|
||||
cu_num_logits = np.array([0, 3, 4, 11, 13], dtype=np.int32)
|
||||
|
||||
@@ -33,28 +74,11 @@ def test_chunked_scores_match_full_batch(logprobs_mode: str):
|
||||
cu_num_logits_np = np.array([0, 3, 4, 8, 10], dtype=np.int32)
|
||||
num_logits_per_req = np.diff(cu_num_logits_np)
|
||||
idx_mapping_np = np.array([7, 2, 9, 1], dtype=np.int32)
|
||||
input_batch = SimpleNamespace(
|
||||
num_reqs=4,
|
||||
cu_num_logits_np=cu_num_logits_np,
|
||||
cu_num_logits=torch.from_numpy(cu_num_logits_np).to(device),
|
||||
idx_mapping_np=idx_mapping_np,
|
||||
idx_mapping=torch.from_numpy(idx_mapping_np).to(device),
|
||||
expanded_idx_mapping=torch.from_numpy(
|
||||
np.repeat(idx_mapping_np, num_logits_per_req)
|
||||
).to(device),
|
||||
expanded_local_pos=torch.from_numpy(
|
||||
np.concatenate(
|
||||
[np.arange(count, dtype=np.int32) for count in num_logits_per_req]
|
||||
)
|
||||
).to(device),
|
||||
seq_lens_cpu_upper_bound=torch.from_numpy(
|
||||
np.array([10, 20, 30, 40], dtype=np.int32)
|
||||
),
|
||||
input_batch = _make_input_batch(cu_num_logits_np, idx_mapping_np, device)
|
||||
rejection_sampler = _make_rejection_sampler(
|
||||
SimpleNamespace(logprobs_mode=logprobs_mode, return_sampling_mask=False),
|
||||
num_speculative_steps=3,
|
||||
)
|
||||
rejection_sampler = object.__new__(RejectionSampler)
|
||||
rejection_sampler.sampler = SimpleNamespace(logprobs_mode=logprobs_mode)
|
||||
rejection_sampler.num_speculative_steps = 3
|
||||
rejection_sampler.enable_adaptive_verification = False
|
||||
|
||||
def fake_verify(
|
||||
self,
|
||||
@@ -75,14 +99,16 @@ def test_chunked_scores_match_full_batch(logprobs_mode: str):
|
||||
rejection_sampler._verify = MethodType(fake_verify, rejection_sampler)
|
||||
logits = torch.arange(170, dtype=torch.float32, device=device).view(10, 17)
|
||||
|
||||
sampled, num_sampled, chunked_logprobs = rejection_sampler._verify_in_chunks(
|
||||
logits,
|
||||
input_batch,
|
||||
draft_logits=None,
|
||||
draft_sampled=torch.arange(10, device=device),
|
||||
pos=torch.arange(10, device=device),
|
||||
max_chunk_logits=5,
|
||||
max_num_logprobs=2,
|
||||
sampled, num_sampled, chunked_logprobs, sampling_mask_tensors = (
|
||||
rejection_sampler._verify_in_chunks(
|
||||
logits,
|
||||
input_batch,
|
||||
draft_logits=None,
|
||||
draft_sampled=torch.arange(10, device=device),
|
||||
pos=torch.arange(10, device=device),
|
||||
max_chunk_logits=5,
|
||||
max_num_logprobs=2,
|
||||
)
|
||||
)
|
||||
score_logits = logits + 1 if logprobs_mode in PROCESSED_LOGPROBS_MODES else logits
|
||||
full_logprobs = rejection_sampler._get_logprobs_tensors(
|
||||
@@ -111,3 +137,155 @@ def test_chunked_scores_match_full_batch(logprobs_mode: str):
|
||||
chunked_logprobs.cu_num_generated_tokens
|
||||
== full_logprobs.cu_num_generated_tokens
|
||||
)
|
||||
assert sampling_mask_tensors is None
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
|
||||
def test_replay_on_off_preserves_rejection_sampling_and_rng(monkeypatch):
|
||||
device = torch.device("cuda")
|
||||
num_reqs = 3
|
||||
num_speculative_steps = 3
|
||||
rows_per_request = num_speculative_steps + 1
|
||||
vocab_size = 8
|
||||
cu_num_logits_np = np.arange(0, 13, rows_per_request, dtype=np.int32)
|
||||
idx_mapping_np = np.arange(num_reqs, dtype=np.int32)
|
||||
input_batch = _make_input_batch(cu_num_logits_np, idx_mapping_np, device)
|
||||
processed_logits = torch.zeros((12, vocab_size), device=device)
|
||||
processed_logits[0, 0] = -float("inf")
|
||||
processed_logits[rows_per_request + 1, 1] = -float("inf")
|
||||
draft_logits = torch.zeros(
|
||||
(num_reqs, num_speculative_steps, vocab_size), device=device
|
||||
)
|
||||
draft_sampled = torch.zeros(
|
||||
(num_reqs, rows_per_request), dtype=torch.int64, device=device
|
||||
)
|
||||
draft_sampled[:, 1:] = torch.tensor([0, 1, 2], device=device)
|
||||
draft_sampled = draft_sampled.flatten()
|
||||
sampler = SimpleNamespace(
|
||||
logprobs_mode="processed_logprobs",
|
||||
return_sampling_mask=False,
|
||||
sampling_states=SimpleNamespace(
|
||||
top_k=SimpleNamespace(np=np.full(num_reqs, vocab_size)),
|
||||
temperature=SimpleNamespace(gpu=torch.ones(num_reqs, device=device)),
|
||||
seeds=SimpleNamespace(
|
||||
gpu=torch.arange(num_reqs, dtype=torch.int64, device=device)
|
||||
),
|
||||
),
|
||||
use_fp64_gumbel=False,
|
||||
apply_sampling_params=lambda logits, *_: logits,
|
||||
)
|
||||
rejection_sampler = _make_rejection_sampler(sampler, num_speculative_steps)
|
||||
pos = torch.arange(12, dtype=torch.int32, device=device)
|
||||
|
||||
pack_calls: list[tuple[list[int], int, int]] = []
|
||||
pack = SamplingMaskTensors.from_logits.__func__
|
||||
|
||||
def track_pack(
|
||||
cls,
|
||||
logits,
|
||||
cu_num_logits,
|
||||
num_sampled_tokens,
|
||||
max_num_kept,
|
||||
rows_per_request=1,
|
||||
):
|
||||
pack_calls.append(
|
||||
(cu_num_logits.cpu().tolist(), max_num_kept, rows_per_request)
|
||||
)
|
||||
return pack(
|
||||
cls,
|
||||
logits,
|
||||
cu_num_logits,
|
||||
num_sampled_tokens,
|
||||
max_num_kept,
|
||||
rows_per_request,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
SamplingMaskTensors,
|
||||
"from_logits",
|
||||
classmethod(track_pack),
|
||||
)
|
||||
|
||||
rng_before = torch.cuda.get_rng_state()
|
||||
off_sampled, off_counts, off_logprobs, off_masks = (
|
||||
rejection_sampler._verify_in_chunks(
|
||||
processed_logits,
|
||||
input_batch,
|
||||
draft_logits,
|
||||
draft_sampled,
|
||||
pos,
|
||||
max_chunk_logits=8,
|
||||
max_num_logprobs=0,
|
||||
)
|
||||
)
|
||||
rng_after_off = torch.cuda.get_rng_state()
|
||||
assert pack_calls == []
|
||||
|
||||
sampler.return_sampling_mask = True
|
||||
on_sampled, on_counts, on_logprobs, on_masks = rejection_sampler._verify_in_chunks(
|
||||
processed_logits,
|
||||
input_batch,
|
||||
draft_logits,
|
||||
draft_sampled,
|
||||
pos,
|
||||
max_chunk_logits=8,
|
||||
max_num_logprobs=0,
|
||||
)
|
||||
rng_after_on = torch.cuda.get_rng_state()
|
||||
|
||||
assert pack_calls == [
|
||||
([0, 4, 8], vocab_size, rows_per_request),
|
||||
([0, 4], vocab_size, rows_per_request),
|
||||
]
|
||||
assert off_masks is None and on_masks is not None
|
||||
assert on_masks.rows_per_request == rows_per_request
|
||||
assert on_masks.token_ids.shape == (num_reqs * rows_per_request, vocab_size)
|
||||
assert on_masks.packed_mask.shape[0] == num_reqs * rows_per_request
|
||||
assert off_counts.tolist() == [1, 2, 4]
|
||||
assert torch.equal(off_counts, on_counts)
|
||||
counts = off_counts.cpu().tolist()
|
||||
for req_idx, count in enumerate(counts):
|
||||
assert torch.equal(off_sampled[req_idx, :count], on_sampled[req_idx, :count])
|
||||
|
||||
mapped_rows = [
|
||||
int(input_batch.cu_num_logits_np[req_idx]) + slot_idx
|
||||
for req_idx, count in enumerate(counts)
|
||||
for slot_idx in range(count)
|
||||
]
|
||||
assert off_logprobs is not None and on_logprobs is not None
|
||||
assert torch.equal(
|
||||
off_logprobs.logprob_token_ids[mapped_rows],
|
||||
on_logprobs.logprob_token_ids[mapped_rows],
|
||||
)
|
||||
assert torch.equal(
|
||||
off_logprobs.logprobs[mapped_rows], on_logprobs.logprobs[mapped_rows]
|
||||
)
|
||||
assert torch.equal(
|
||||
off_logprobs.selected_token_ranks[mapped_rows],
|
||||
on_logprobs.selected_token_ranks[mapped_rows],
|
||||
)
|
||||
assert torch.equal(rng_before, rng_after_off)
|
||||
assert torch.equal(rng_after_off, rng_after_on)
|
||||
|
||||
masks = (
|
||||
on_masks.to_cpu_nonblocking().tolists(on_counts.cpu().numpy()).to_nested_list()
|
||||
)
|
||||
expected_masks = [
|
||||
torch.isfinite(processed_logits[row]).nonzero().flatten().tolist()
|
||||
for row in mapped_rows
|
||||
]
|
||||
emitted = [
|
||||
int(on_sampled[req_idx, slot_idx])
|
||||
for req_idx, count in enumerate(on_counts.cpu().tolist())
|
||||
for slot_idx in range(count)
|
||||
]
|
||||
expected_logprobs = torch.stack(
|
||||
[
|
||||
torch.log_softmax(processed_logits[row], dim=-1)[token_id]
|
||||
for row, token_id in zip(mapped_rows, emitted)
|
||||
]
|
||||
)
|
||||
|
||||
assert masks == expected_masks
|
||||
assert all(token_id in mask for token_id, mask in zip(emitted, masks))
|
||||
assert torch.equal(on_logprobs.logprobs[mapped_rows, 0], expected_logprobs)
|
||||
|
||||
Reference in New Issue
Block a user