From 0f8b398158f01fb3ea0e81b5adc93338fdf3db9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Rialland?= <36076211+TQCB@users.noreply.github.com> Date: Wed, 30 Sep 2026 18:22:10 +0200 Subject: [PATCH] [watermarking] add context deduplication support to speculative decoding (#56807) Signed-off-by: Raphael Rialland Signed-off-by: Simon Veitner Co-authored-by: Simon Veitner Co-authored-by: Lucas Wilkinson --- docs/features/watermarking.md | 29 +- tests/test_config.py | 25 +- .../test_rejection_sampler_utils.py | 176 ++++++ tests/watermarking/test_gumbel.py | 62 ++ tests/watermarking/test_watermarking.py | 563 +++++++++++++++++- vllm/config/vllm.py | 7 - vllm/v1/watermarking/gpu_sampler.py | 13 +- vllm/v1/watermarking/gumbel.py | 4 + vllm/v1/watermarking/spec_decode.py | 122 +++- vllm/v1/watermarking/watermarker.py | 9 +- vllm/v1/worker/gpu/model_runner.py | 6 +- vllm/v1/worker/gpu/sample/gumbel.py | 30 +- vllm/v1/worker/gpu/sample/watermark.py | 322 +++++++++- .../gpu/spec_decode/dspark/speculator.py | 12 +- .../gpu/spec_decode/rejection_sampler.py | 13 +- .../spec_decode/rejection_sampler_utils.py | 47 +- vllm/v1/worker/gpu/spec_decode/speculator.py | 27 +- 17 files changed, 1379 insertions(+), 88 deletions(-) diff --git a/docs/features/watermarking.md b/docs/features/watermarking.md index 54c7e58ce7..7ab02d3363 100644 --- a/docs/features/watermarking.md +++ b/docs/features/watermarking.md @@ -77,30 +77,17 @@ watermarked. The watermark signal is diluted in proportion to the share of output tokens supplied by accepted drafts; rejected drafts do not dilute it because their recovery tokens are watermarked. -Speculative-decoding token paths do not currently support generation-side -context deduplication. The configured `deduplicate_contexts` policy is not -applied to accepted drafts, rejection-recovery tokens, or bonus tokens. - For `dual_key_gumbel`, `alpha` has no effect under speculative decoding. The speculative protocol selects the key for each token instead. -## Algorithms - -### Gumbel-max - -Gumbel-max derives a deterministic pseudorandom value from the key, prior token -context, and every candidate token, then uses the resulting Gumbel noise for -categorical sampling. See -[Aaronson's original presentation](https://simons.berkeley.edu/sites/default/files/2024-10/LLM24-2%20Slides%20-%20Scott%20Aaronson.pdf). - -Gumbel-max requires stochastic sampling. Greedy requests (`temperature=0`) -bypass watermarking and emit a warning once per worker. +## Context deduplication When a token context is repeated, generation can use ordinary sampling for that occurrence. Reusing the repeated context leads to a bias over the sequence, as certain token choices would be correlated. Ordinary sampling at these positions allows for single-sequence non-distortion (see section G.3 of the [SynthID-Text supplementary materials](https://media.springernature.com/original/springer-static/esm/art%3A10.1038%2Fs41586-024-08025-4/MediaObjects/41586_2024_8025_MOESM1_ESM.pdf)). +This also applies to drafter watermarking during speculative decoding. The detector independently deduplicates contexts so repeated keyed random vectors are not treated as independent evidence, meaning that context deduplication at generation time does not reduce the watermarking signal, unless @@ -143,6 +130,18 @@ vllm serve MODEL \ '{"algorithm":"gumbel","key":42,"deduplicate_contexts":"all"}' ``` +## Algorithms + +### Gumbel-max + +Gumbel-max derives a deterministic pseudorandom value from the key, prior token +context, and every candidate token, then uses the resulting Gumbel noise for +categorical sampling. See +[Aaronson's original presentation](https://simons.berkeley.edu/sites/default/files/2024-10/LLM24-2%20Slides%20-%20Scott%20Aaronson.pdf). + +Gumbel-max requires stochastic sampling. Greedy requests (`temperature=0`) +bypass watermarking and emit a warning once per worker. + ### Dual-key Gumbel-max Dual-key Gumbel-max derives independent keys A and B from one configured master diff --git a/tests/test_config.py b/tests/test_config.py index 30161e89fb..93578ab918 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -3526,7 +3526,6 @@ def test_target_only_gumbel_allows_speculative_decoding(caplog_vllm, disable_log config._check_watermarking_unsupported() assert "Target-only watermarking leaves accepted draft tokens" in caplog_vllm.text - assert "Context deduplication is not supported" in caplog_vllm.text def test_speculative_watermarking_without_context_dedup_does_not_warn( @@ -3543,10 +3542,32 @@ def test_speculative_watermarking_without_context_dedup_does_not_warn( parallel_drafting=False, ) + caplog_vllm.clear() with caplog_vllm.at_level(logging.WARNING): config._check_watermarking_unsupported() - assert "Context deduplication is not supported" not in caplog_vllm.text + assert "dedup" not in caplog_vllm.text.lower() + + +def test_speculative_context_dedup_logs_no_unsupported_warning( + caplog_vllm, disable_log_dedup +): + config = _watermarked_vllm_config() + config.watermark_config = WatermarkConfig( + algorithm="dual_key_gumbel", key=42, deduplicate_contexts="single_turn" + ) + config.speculative_config = SimpleNamespace( + method="mtp", + draft_sample_method="probabilistic", + rejection_sample_method="standard", + parallel_drafting=False, + ) + + caplog_vllm.clear() + with caplog_vllm.at_level(logging.WARNING): + config._check_watermarking_unsupported() + + assert "dedup" not in caplog_vllm.text.lower() def test_gumbel_rejects_speculative_decoding_without_target_only(): diff --git a/tests/v1/spec_decode/test_rejection_sampler_utils.py b/tests/v1/spec_decode/test_rejection_sampler_utils.py index 7fc6898142..e80f9e6d2c 100644 --- a/tests/v1/spec_decode/test_rejection_sampler_utils.py +++ b/tests/v1/spec_decode/test_rejection_sampler_utils.py @@ -203,6 +203,182 @@ def test_watermarked_recovery_uses_target_key(): assert sampled[0, 0] == expected.token_ids[0] +def test_repeated_draft_context_does_not_change_acceptance(): + torch.manual_seed(0) + num_trials = 8 + # A draft distribution of its own, so the drafts are not accepted outright. + target_logits = torch.randn(64, device="cuda") + draft_logits = torch.randn(64, device="cuda") + inputs = _build_rejection_sample_inputs( + target_logits, + draft_logits, + num_speculative_steps=1, + temperature=1.0, + num_trials=num_trials, + ) + expected, expected_num_sampled = rejection_sample(**inputs, num_speculative_steps=1) + + sampled, num_sampled = watermarked_rejection_sample( + **inputs, + num_speculative_steps=1, + contexts=torch.tensor([[10, 11], [11, 12]], device="cuda").repeat( + num_trials, 1 + ), + watermarking=torch.ones(num_trials, dtype=torch.bool, device="cuda"), + watermarking_skip_mask=torch.tensor([True, False], device="cuda").repeat( + num_trials + ), + watermarker=GumbelWatermarker(key=42, context_width=2), + ) + + # Some drafts must actually be rejected, or the ordinary-residual branch + # the skip mask selects is never reached. + assert (expected_num_sampled == 1).any() + torch.testing.assert_close(num_sampled, expected_num_sampled, rtol=0, atol=0) + torch.testing.assert_close(sampled[:, :1], expected[:, :1], rtol=0, atol=0) + + +def test_repeated_context_skip_mask_is_read_per_row_in_a_block(): + """A block of three drafts, accepted at row 0 and rejected at row 1. + + The skip mask selects the sampler per row, so the row the target recovers + at decides which key draws the recovery token. + """ + num_speculative_steps = 3 + target_logits = torch.tensor([float("-inf"), 0.0, 1.0, 2.0], device="cuda") + draft_logits = torch.tensor([30.0, -30.0, -30.0, -30.0], device="cuda") + inputs = _build_rejection_sample_inputs( + target_logits, + draft_logits, + num_speculative_steps=num_speculative_steps, + temperature=1.0, + num_trials=1, + ) + # Row 0 drafts a token the target gives 0.67 and the draft e**-60, so it is + # accepted; row 1 drafts a token the target gives no mass, so it is not. + inputs["draft_sampled"][1] = 3 + inputs["draft_sampled"][2] = 0 + contexts = torch.tensor( + [[10, 11], [11, 3], [3, 0], [0, 1]], dtype=torch.int64, device="cuda" + ) + expected, expected_num_sampled = rejection_sample( + **inputs, num_speculative_steps=num_speculative_steps + ) + assert expected_num_sampled.item() == 2 + assert expected[0, 0].item() == 3 + + watermarker = GumbelWatermarker(key=42, context_width=2) + skipped, skipped_num_sampled = watermarked_rejection_sample( + **inputs, + num_speculative_steps=num_speculative_steps, + contexts=contexts, + watermarking=torch.tensor([True], device="cuda"), + watermarking_skip_mask=torch.tensor([False, True, False, False], device="cuda"), + watermarker=watermarker, + ) + + assert skipped_num_sampled.item() == 2 + torch.testing.assert_close(skipped[:, :2], expected[:, :2], rtol=0, atol=0) + + watermarked, watermarked_num_sampled = watermarked_rejection_sample( + **inputs, + num_speculative_steps=num_speculative_steps, + contexts=contexts, + watermarking=torch.tensor([True], device="cuda"), + watermarking_skip_mask=torch.zeros( + num_speculative_steps + 1, dtype=torch.bool, device="cuda" + ), + watermarker=watermarker, + ) + # The residual equals the target to within e**-60, so the recovery token is + # the target key's draw on the rejected row's context. + recovery = GumbelWatermarker(key=42, context_width=2).sample( + target_logits.unsqueeze(0), contexts[1:2], lambda _: None + ) + + assert watermarked_num_sampled.item() == 2 + assert watermarked[0, 0].item() == 3 + assert watermarked[0, 1] == recovery.token_ids[0] + + +def test_repeated_recovery_context_uses_ordinary_residual_sample(): + target_logits = torch.tensor([float("-inf"), 0.0, 1.0, 2.0], device="cuda") + draft_logits = torch.tensor([100.0, -100.0, -100.0, -100.0], device="cuda") + inputs = _build_rejection_sample_inputs( + target_logits, + draft_logits, + num_speculative_steps=1, + temperature=1.0, + num_trials=1, + ) + inputs["draft_sampled"][1] = 0 + expected, expected_num_sampled = rejection_sample(**inputs, num_speculative_steps=1) + + sampled, num_sampled = watermarked_rejection_sample( + **inputs, + num_speculative_steps=1, + contexts=torch.tensor([[10, 11], [11, 0]], device="cuda"), + watermarking=torch.tensor([True], device="cuda"), + watermarking_skip_mask=torch.tensor([True, False], device="cuda"), + watermarker=GumbelWatermarker(key=42, context_width=2), + ) + + torch.testing.assert_close(num_sampled, expected_num_sampled, rtol=0, atol=0) + torch.testing.assert_close(sampled[:, :1], expected[:, :1], rtol=0, atol=0) + + +def test_repeated_bonus_context_uses_ordinary_target_sample(): + target_logits = torch.randn(64, device="cuda") + inputs = _build_rejection_sample_inputs( + target_logits, + target_logits, + num_speculative_steps=1, + temperature=1.0, + num_trials=1, + ) + expected, expected_num_sampled = rejection_sample(**inputs, num_speculative_steps=1) + + sampled, num_sampled = watermarked_rejection_sample( + **inputs, + num_speculative_steps=1, + contexts=torch.tensor([[10, 11], [11, 12]], device="cuda"), + watermarking=torch.tensor([True], device="cuda"), + watermarking_skip_mask=torch.tensor([False, True], device="cuda"), + watermarker=GumbelWatermarker(key=42, context_width=2), + ) + + torch.testing.assert_close(num_sampled, expected_num_sampled, rtol=0, atol=0) + torch.testing.assert_close(sampled[:, :2], expected[:, :2], rtol=0, atol=0) + + +def test_repeated_context_does_not_change_unwatermarked_or_greedy_rows(): + target_logits = torch.randn(64, device="cuda") + inputs = _build_rejection_sample_inputs( + target_logits, + target_logits, + num_speculative_steps=1, + temperature=1.0, + num_trials=2, + ) + inputs["temperature"][1] = 0 + expected, expected_num_sampled = rejection_sample(**inputs, num_speculative_steps=1) + + sampled, num_sampled = watermarked_rejection_sample( + **inputs, + num_speculative_steps=1, + contexts=torch.tensor([[10, 11], [11, 12], [20, 21], [21, 22]], device="cuda"), + watermarking=torch.tensor([False, True], device="cuda"), + watermarking_skip_mask=torch.ones(4, dtype=torch.bool, device="cuda"), + watermarker=GumbelWatermarker(key=42, context_width=2), + ) + + torch.testing.assert_close(num_sampled, expected_num_sampled, rtol=0, atol=0) + for row, length in enumerate(num_sampled): + torch.testing.assert_close( + sampled[row, : int(length)], expected[row, : int(length)], rtol=0, atol=0 + ) + + def test_watermarked_recovery_supports_smaller_draft_vocabulary(): target_logits = torch.tensor([float("-inf"), 0.0, 1.0, 2.0, 100.0], device="cuda") draft_logits = torch.tensor([100.0, -100.0, -100.0, -100.0], device="cuda") diff --git a/tests/watermarking/test_gumbel.py b/tests/watermarking/test_gumbel.py index dd32bda627..d789692a0e 100644 --- a/tests/watermarking/test_gumbel.py +++ b/tests/watermarking/test_gumbel.py @@ -12,6 +12,7 @@ from vllm.v1.watermarking import ( derive_watermark_key, ) from vllm.v1.watermarking.gumbel import _gamma_survival_integer_shape +from vllm.v1.watermarking.watermarker import RandomSampler from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample from vllm.v1.worker.gpu.sample.watermark import philox_gumbel_sample @@ -197,3 +198,64 @@ def test_philox_gumbel_sample_skip_mask_matches_separate_samplers(use_fp64: bool ) torch.testing.assert_close(actual, expected, rtol=0, atol=0) + + +@pytest.mark.skipif( + not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator" +) +def test_philox_mixed_draft_sampling_caches_unprocessed_logits(): + torch.manual_seed(0) + num_rows, vocab_size, num_steps = 5, 4099, 3 + logits = torch.randn(num_rows, vocab_size, device="cuda", dtype=torch.float16) + temperatures = torch.tensor([0.0, 0.3, 0.5, 1.0, 2.0], device="cuda") + processed_logits = logits / torch.where( + temperatures == 0, 1, temperatures + ).unsqueeze(-1) + contexts = torch.randint(0, vocab_size, (num_rows, 4), device="cuda") + idx_mapping = torch.arange(num_rows, dtype=torch.int32, device="cuda") + seeds = torch.arange(num_rows, dtype=torch.int64, device="cuda") + 10 + positions = torch.arange(num_rows, dtype=torch.int64, device="cuda") + 20 + # The skipped rows must divide by a temperature other than 1, or the + # unprocessed logits the cache stores are the logits the sampler sees. + # Row 0 keeps a greedy row in the skipped set. + skip_mask = torch.tensor([True, True, False, False, True], device="cuda") + cols = torch.arange(num_rows, dtype=torch.int32, device="cuda") % num_steps + cache = torch.zeros( + num_rows, num_steps, vocab_size + 1, dtype=logits.dtype, device="cuda" + ) + + ordinary = gumbel_sample( + logits, + idx_mapping, + temperatures, + seeds, + positions, + apply_temperature=True, + is_drafting=True, + ) + watermarked = philox_gumbel_sample(processed_logits, contexts, 42) + expected = torch.where(skip_mask, ordinary, watermarked) + actual = ( + GumbelWatermarker(key=42, context_width=4) + .sample( + processed_logits, + contexts, + random_sampler=RandomSampler( + expanded_idx_mapping=idx_mapping, + temperatures=temperatures, + seeds=seeds, + positions=positions, + is_drafting=True, + logits_cache=cache, + logits_cache_col=cols, + logits_cache_source=logits, + ), + skip_mask=skip_mask, + ) + .token_ids + ) + + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + stored = cache[torch.arange(num_rows, device="cuda"), cols.long(), :vocab_size] + assert torch.equal(stored.view(torch.int16), logits.view(torch.int16)) + assert not cache[:, :, -1].any() diff --git a/tests/watermarking/test_watermarking.py b/tests/watermarking/test_watermarking.py index 232f19a9a6..03c01b2485 100644 --- a/tests/watermarking/test_watermarking.py +++ b/tests/watermarking/test_watermarking.py @@ -28,6 +28,7 @@ from vllm.v1.watermarking.spec_decode import ( from vllm.v1.watermarking.watermarker import Watermarker, WatermarkSample from vllm.v1.worker.gpu.sample.sampler import Sampler from vllm.v1.worker.gpu.sample.watermark import ( + draft_watermarking_mask, philox_gumbel_sample, repeated_context_mask, ) @@ -202,11 +203,17 @@ def test_speculative_decoding_uses_fixed_dual_key_roles(): max_num_reqs=1, device=torch.device("cpu"), allow_target_only=False, + num_speculative_steps=3, + deduplicate_contexts="all", + deduplicate_contexts_max_history=None, ) assert target.prf.key == derive_watermark_key(42, b"key_b") assert draft is not None assert draft.watermarker.prf.key == derive_watermark_key(42, b"key_a") + assert draft.num_speculative_steps == 3 + assert draft.deduplicate_contexts == "all" + assert draft.deduplicate_contexts_max_history is None def test_recovery_key_rejects_the_unsplit_dual_key_watermarker(): @@ -892,6 +899,187 @@ def test_gpu_sampler_builds_speculative_contexts_from_drafts(): assert torch.equal(contexts, torch.tensor([[10, 11], [11, 20], [20, 21]])) +def test_gpu_sampler_deduplicates_contexts_within_speculative_block(): + sampler = object.__new__(GPUWatermarkSampler) + sampler.watermarker = SimpleNamespace(context_width=2) + sampler.deduplicate_contexts = "single_turn" + sampler.deduplicate_contexts_max_history = None + sampler.num_speculative_tokens = 4 + sampler.req_states = SimpleNamespace( + all_token_ids=SimpleNamespace(gpu=torch.tensor([[100, 1, 2, 0, 0, 0]])), + prompt_len=SimpleNamespace(gpu=torch.tensor([1])), + total_len=SimpleNamespace(gpu=torch.tensor([3])), + ) + request_indices = torch.tensor([0, 0, 0, 0, 0]) + local_positions = torch.tensor([0, 1, 2, 3, 4]) + contexts = sampler._get_contexts( + request_indices, + local_positions, + torch.tensor([0, 3, 4, 1, 2]), + ) + + repeated = sampler._get_repeated_contexts( + request_indices, contexts, local_positions + ) + + assert torch.equal( + contexts, + torch.tensor([[1, 2], [2, 3], [3, 4], [4, 1], [1, 2]]), + ) + assert torch.equal(repeated, torch.tensor([False, False, False, False, True])) + + sampler.deduplicate_contexts_max_history = 3 + repeated = sampler._get_repeated_contexts( + request_indices, contexts, local_positions + ) + assert not repeated.any() + + +def test_gpu_sampler_all_scope_skips_only_partial_speculative_contexts(): + sampler = object.__new__(GPUWatermarkSampler) + sampler.watermarker = SimpleNamespace(context_width=2) + sampler.deduplicate_contexts = "all" + sampler.deduplicate_contexts_max_history = None + sampler.num_speculative_tokens = 3 + sampler.req_states = SimpleNamespace( + all_token_ids=SimpleNamespace(gpu=torch.tensor([[100, 101, 10, 0, 0]])), + prompt_len=SimpleNamespace(gpu=torch.tensor([2])), + total_len=SimpleNamespace(gpu=torch.tensor([3])), + ) + request_indices = torch.tensor([0, 0, 0]) + local_positions = torch.tensor([0, 1, 2]) + contexts = sampler._get_contexts( + request_indices, + local_positions, + torch.tensor([10, 20, 30]), + ) + + repeated = sampler._get_repeated_contexts( + request_indices, contexts, local_positions + ) + + assert torch.equal(contexts, torch.tensor([[-1, 10], [10, 20], [20, 30]])) + assert torch.equal(repeated, torch.tensor([True, False, False])) + + +@pytest.mark.skipif( + not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator" +) +@pytest.mark.parametrize("max_history,expected_last", [(None, True), (3, False)]) +def test_speculative_context_repetition_accelerator_parity( + max_history: int | None, expected_last: bool +): + all_token_ids = torch.tensor([[10, 11]]) + req_indices = torch.tensor([0, 0, 0, 0, 0]) + prompt_lens = torch.tensor([0]) + total_lens = torch.tensor([2]) + local_positions = torch.tensor([0, 1, 2, 3, 4]) + contexts = torch.tensor([[1, 2], [2, 3], [3, 4], [4, 1], [1, 2]]) + + expected = repeated_context_mask( + all_token_ids, + req_indices, + prompt_lens, + total_lens, + contexts, + history_offsets=local_positions, + local_positions=local_positions, + num_speculative_steps=4, + max_history=max_history, + ) + actual = repeated_context_mask( + all_token_ids.cuda(), + req_indices.cuda(), + prompt_lens.cuda(), + total_lens.cuda(), + contexts.cuda(), + history_offsets=local_positions.cuda(), + local_positions=local_positions.cuda(), + num_speculative_steps=4, + max_history=max_history, + ).cpu() + + assert torch.equal( + expected, torch.tensor([False, False, False, False, expected_last]) + ) + assert torch.equal(actual, expected) + + +@pytest.mark.skipif( + not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator" +) +@pytest.mark.parametrize("steps_layout", ["per_row", "broadcast"]) +@pytest.mark.parametrize("req_indices_layout", ["contiguous", "column_view"]) +def test_draft_context_deduplication_accelerator_parity( + steps_layout, req_indices_layout +): + """Parity with the CPU reference for the layouts the speculators pass. + + A draft step is one 0-d tensor broadcast over the batch, and DSpark samples + a step at a time, so its request indices are an `idx_map[:, i]` column. Both + are read per row, so a row that ignores the layout consults another row's + step or another request's history. + """ + num_rows, num_steps, step = 4, 3, 1 + all_token_ids = torch.tensor([[10, 11, 20, 11], [30, 31, 11, 20]]) + prompt_lens = torch.tensor([0, 0]) + total_lens = torch.tensor([4, 4]) + contexts = torch.tensor([[40, 41], [31, 11], [10, 11], [50, 51]]) + prior_contexts = torch.zeros(num_rows, num_steps, 2, dtype=torch.int64) + prior_contexts[0, 0] = torch.tensor([40, 41]) + enabled = torch.ones(num_rows, dtype=torch.bool) + req_indices = torch.tensor([0, 1, 0, 1]) + + def run(all_token_ids, req_indices, steps, prior): + return draft_watermarking_mask( + all_token_ids, + req_indices, + prompt_lens.to(all_token_ids.device), + total_lens.to(all_token_ids.device), + prior, + contexts.to(all_token_ids.device), + steps, + enabled.to(all_token_ids.device), + max_history=3, + include_prompt=False, + ) + + reference_prior = prior_contexts.clone() + reference = run( + all_token_ids, + req_indices, + torch.full((num_rows,), step), + reference_prior, + ) + + # Row 0 repeats its own step-0 context, rows 1 and 2 repeat a committed + # context of their own request, row 3 repeats nothing. + assert torch.equal(reference, torch.tensor([False, False, False, True])) + expected_prior = prior_contexts.clone() + expected_prior[:, step] = contexts + assert torch.equal(reference_prior, expected_prior) + + if steps_layout == "per_row": + steps = torch.full((num_rows,), step).cuda() + else: + steps = torch.tensor(step).cuda().expand(num_rows) + assert steps.stride(0) == 0 + + if req_indices_layout == "contiguous": + gpu_req_indices = req_indices.cuda() + else: + gpu_req_indices = ( + req_indices.repeat_interleave(num_steps).view(num_rows, num_steps).cuda() + )[:, 1] + assert gpu_req_indices.stride(0) == num_steps + + actual_prior = prior_contexts.cuda() + actual = run(all_token_ids.cuda(), gpu_req_indices, steps, actual_prior).cpu() + + assert torch.equal(actual, reference) + assert torch.equal(actual_prior.cpu(), expected_prior) + + def test_gpu_sampler_builds_chunked_multi_request_speculative_contexts(): sampler = object.__new__(GPUWatermarkSampler) sampler.watermarker = SimpleNamespace(context_width=2) @@ -921,6 +1109,69 @@ def test_gpu_sampler_builds_chunked_multi_request_speculative_contexts(): ) +@pytest.mark.skipif( + not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator" +) +def test_draft_context_deduplication_broadcasts_the_draft_step(): + """The draft step is one 0-d tensor broadcast over the batch. + + The context occurs four positions back, which `max_history - step` puts + just outside the scanned window, so a row that mistakes another row's step + for its own reports the opposite verdict. The neighbours leg is the + deterministic control: a row that reads a step of 0 instead of 2 widens its + window by two positions and finds the context. The production leg asserts + the layout the speculators pass, where the stride-less read lands outside + the allocation and can return the right value by chance. + """ + num_rows = 4 + all_token_ids = torch.tensor([[3, 4, 9, 9, 9, 9], [3, 4, 9, 9, 9, 9]]) + req_indices = torch.tensor([0, 0, 1, 1]) + prompt_lens = torch.tensor([0, 0]) + total_lens = torch.tensor([6, 6]) + prior_contexts = torch.zeros(num_rows, 3, 2, dtype=torch.int64) + contexts = torch.tensor([[3, 4]]).expand(num_rows, 2).contiguous() + enabled = torch.ones(num_rows, dtype=torch.bool) + + def run(steps, device): + prior = prior_contexts.clone().to(device) + active = draft_watermarking_mask( + all_token_ids.to(device), + req_indices.to(device), + prompt_lens.to(device), + total_lens.to(device), + prior, + contexts.to(device), + steps, + enabled.to(device), + max_history=4, + include_prompt=False, + ) + return active.cpu(), prior.cpu() + + expected, expected_prior = run(torch.full((num_rows,), 2), torch.device("cpu")) + assert torch.equal(expected, torch.ones(num_rows, dtype=torch.bool)) + + materialized, materialized_prior = run( + torch.full((num_rows,), 2).cuda(), torch.device("cuda") + ) + assert torch.equal(materialized, expected) + assert torch.equal(materialized_prior, expected_prior) + + # Neighbouring steps are the value a row reads when the broadcast stride is + # ignored; the production tensor reads uninitialized memory instead. + neighbours = torch.tensor([2, 0, 0, 0]).cuda() + broadcast, broadcast_prior = run( + neighbours[0].expand(num_rows), torch.device("cuda") + ) + assert torch.equal(broadcast, expected) + assert torch.equal(broadcast_prior, expected_prior) + + step = torch.tensor(2).cuda() + production, production_prior = run(step.expand(num_rows), torch.device("cuda")) + assert torch.equal(production, expected) + assert torch.equal(production_prior, expected_prior) + + def test_draft_sampler_uses_draft_key_and_advances_context(monkeypatch): class StubSpeculator(DraftModelSpeculator): def capture(self): ... @@ -937,22 +1188,32 @@ def test_draft_sampler_uses_draft_key_and_advances_context(monkeypatch): return torch.zeros(hidden_states.shape[0], 8) class StubWatermarker: + context_width = 2 + @staticmethod - def sample(logits, contexts, random_sampler=None): - assert random_sampler is None - return WatermarkSample(torch.tensor([7, 7]), logits) + def sample(logits, contexts, random_sampler=None, skip_mask=None): + assert random_sampler is not None + return WatermarkSample( + torch.where(skip_mask, random_sampler(logits), 7), logits + ) speculator = object.__new__(StubSpeculator) speculator.model = StubModel() speculator.use_fp64_gumbel = False speculator.acceptance_estimator = None - draft_watermarker = object.__new__(DraftWatermarker) - draft_watermarker.watermarker = StubWatermarker() - draft_watermarker.contexts = torch.tensor([[1, 2], [3, 4]]) + draft_watermarker = DraftWatermarker( + StubWatermarker(), + max_num_reqs=2, + device=torch.device("cpu"), + num_speculative_steps=1, + deduplicate_contexts="none", + deduplicate_contexts_max_history=None, + ) + draft_watermarker.contexts.copy_(torch.tensor([[1, 2], [3, 4]])) draft_watermarker.enabled = torch.tensor([True, False]) speculator.draft_watermarker = draft_watermarker monkeypatch.setattr( - "vllm.v1.worker.gpu.spec_decode.speculator.gumbel_sample", + "vllm.v1.watermarking.watermarker.gumbel_sample", lambda *args, **kwargs: torch.tensor([3, 4]), ) @@ -970,6 +1231,274 @@ def test_draft_sampler_uses_draft_key_and_advances_context(monkeypatch): assert torch.equal(draft_watermarker.contexts, torch.tensor([[2, 7], [4, 4]])) +def test_draft_sampler_uses_ordinary_samples_for_repeated_contexts(monkeypatch): + class StubWatermarker: + context_width = 1 + + def __init__(self): + self.samples = iter((2, 1, 8)) + + def sample(self, logits, contexts, random_sampler=None, skip_mask=None): + watermarked = torch.tensor([next(self.samples)]) + return WatermarkSample( + torch.where(skip_mask, random_sampler(logits), watermarked), logits + ) + + draft_watermarker = DraftWatermarker( + StubWatermarker(), + max_num_reqs=1, + device=torch.device("cpu"), + num_speculative_steps=3, + deduplicate_contexts="single_turn", + deduplicate_contexts_max_history=None, + ) + draft_watermarker.prepare( + contexts=torch.tensor([[1]]), + enabled=torch.tensor([True]), + all_token_ids=torch.tensor([[100, 1, 0, 0]]), + prompt_lens=torch.tensor([1]), + total_lens=torch.tensor([2]), + ) + idx_mapping = torch.tensor([0]) + temperature = torch.tensor([1.0]) + ordinary_samples = iter((torch.tensor([5]), torch.tensor([6]), torch.tensor([4]))) + monkeypatch.setattr( + "vllm.v1.watermarking.watermarker.gumbel_sample", + lambda *args, **kwargs: next(ordinary_samples), + ) + seeds = torch.tensor([0]) + draft_logits = torch.zeros(1, 3, 8) + + first = draft_watermarker.sample( + torch.zeros(1, 8), + idx_mapping, + temperature=temperature, + seed=seeds, + pos=torch.tensor([0]), + apply_temperature=True, + is_drafting=True, + logits_cache_col=torch.tensor(0), + logits_cache=draft_logits, + use_fp64=False, + ) + second = draft_watermarker.sample( + torch.zeros(1, 8), + idx_mapping, + temperature=temperature, + seed=seeds, + pos=torch.tensor([1]), + apply_temperature=True, + is_drafting=True, + logits_cache_col=torch.tensor(1), + logits_cache=draft_logits, + use_fp64=False, + ) + repeated = draft_watermarker.sample( + torch.zeros(1, 8), + idx_mapping, + temperature=temperature, + seed=seeds, + pos=torch.tensor([2]), + apply_temperature=True, + is_drafting=True, + logits_cache_col=torch.tensor(2), + logits_cache=draft_logits, + use_fp64=False, + ) + + assert first.item() == 2 + assert second.item() == 1 + assert repeated.item() == 4 + + +@pytest.mark.skipif( + not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator" +) +def test_draft_context_deduplication_reads_strided_request_indices(): + """DSpark samples a step at a time, so req_indices is an `idx_map[:, i]` column.""" + num_reqs = 4 + num_speculative_steps = 3 + all_token_ids = torch.tensor( + [[3, 4, 3, 4], [9, 9, 9, 9], [3, 4, 3, 4], [9, 9, 9, 9]] + ).cuda() + prompt_lens = torch.zeros(num_reqs, dtype=torch.int64).cuda() + total_lens = torch.full((num_reqs,), 4).cuda() + contexts = torch.tensor([[3, 4]]).expand(num_reqs, 2).contiguous().cuda() + steps = torch.zeros(num_reqs, dtype=torch.int64).cuda() + enabled = torch.ones(num_reqs, dtype=torch.bool).cuda() + + idx_map = ( + torch.arange(num_reqs) + .repeat_interleave(num_speculative_steps) + .view(num_reqs, num_speculative_steps) + .cuda() + ) + column = idx_map[:, 1] + assert column.stride(0) == num_speculative_steps + + def run(req_indices): + prior = torch.zeros( + num_reqs, num_speculative_steps, 2, dtype=torch.int64, device="cuda" + ) + return draft_watermarking_mask( + all_token_ids, + req_indices, + prompt_lens, + total_lens, + prior, + contexts, + steps, + enabled, + max_history=None, + include_prompt=False, + ).cpu() + + expected = torch.tensor([False, True, False, True]) + assert torch.equal(run(column.contiguous()), expected) + assert torch.equal(run(column), expected) + + +def test_draft_sampler_deduplicates_against_committed_history(monkeypatch): + class StubWatermarker: + context_width = 1 + + @staticmethod + def sample(logits, contexts, random_sampler=None, skip_mask=None): + return WatermarkSample( + torch.where(skip_mask, random_sampler(logits), 7), logits + ) + + draft_watermarker = DraftWatermarker( + StubWatermarker(), + max_num_reqs=1, + device=torch.device("cpu"), + num_speculative_steps=1, + deduplicate_contexts="single_turn", + deduplicate_contexts_max_history=None, + ) + draft_watermarker.prepare( + contexts=torch.tensor([[1]]), + enabled=torch.tensor([True]), + all_token_ids=torch.tensor([[100, 1, 2, 1]]), + prompt_lens=torch.tensor([1]), + total_lens=torch.tensor([4]), + ) + + monkeypatch.setattr( + "vllm.v1.watermarking.watermarker.gumbel_sample", + lambda *args, **kwargs: torch.tensor([4]), + ) + sampled = draft_watermarker.sample( + torch.zeros(1, 8), + torch.tensor([0]), + temperature=torch.tensor([1.0]), + seed=torch.tensor([0]), + pos=torch.tensor([0]), + apply_temperature=True, + is_drafting=True, + logits_cache_col=torch.tensor(0), + logits_cache=torch.zeros(1, 1, 8), + use_fp64=False, + ) + + assert sampled.item() == 4 + + +@pytest.mark.skipif( + not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator" +) +@pytest.mark.parametrize("idx_mapping_layout", ["contiguous", "column_view"]) +def test_draft_watermarker_deduplicates_across_a_block_on_cuda( + monkeypatch, idx_mapping_layout +): + """Drive the whole draft sampler over a block with the production shapes. + + `draft_step` is a 0-d tensor, and a speculator that samples one step at a + time hands over a column of its index map. Each step is checked against the + committed history the window still reaches and against the earlier steps of + the block. + """ + num_reqs, num_steps, vocab_size = 2, 3, 16 + watermarker = GumbelWatermarker(key=42, context_width=2) + draft_watermarker = DraftWatermarker( + watermarker, + max_num_reqs=num_reqs, + device=torch.device("cuda"), + num_speculative_steps=num_steps, + deduplicate_contexts="single_turn", + deduplicate_contexts_max_history=4, + ) + draft_watermarker.prepare( + contexts=torch.tensor([[5, 6], [6, 3]]).cuda(), + enabled=torch.ones(num_reqs, dtype=torch.bool).cuda(), + all_token_ids=torch.tensor([[5, 6, 7, 5, 6], [8, 9, 6, 6, 3]]).cuda(), + prompt_lens=torch.zeros(num_reqs, dtype=torch.int64).cuda(), + total_lens=torch.full((num_reqs,), 5).cuda(), + ) + + if idx_mapping_layout == "contiguous": + idx_mapping = torch.arange(num_reqs, dtype=torch.int32).cuda() + else: + idx_mapping = ( + torch.arange(num_reqs, dtype=torch.int32) + .repeat_interleave(num_steps) + .view(num_reqs, num_steps) + .cuda() + )[:, 1] + assert idx_mapping.stride(0) == num_steps + + skip_masks: list[torch.Tensor] = [] + original = philox_gumbel_sample + + def spy(*args, **kwargs): + skip_masks.append(kwargs["skip_mask"].cpu()) + return original(*args, **kwargs) + + monkeypatch.setattr("vllm.v1.worker.gpu.sample.watermark.philox_gumbel_sample", spy) + + draft_logits = torch.zeros(num_reqs, num_steps, vocab_size + 1).cuda() + step_logits = [] + for step in range(num_steps): + # Token 6 wins by a margin no Gumbel draw can close, on either path. + logits = torch.zeros(num_reqs, vocab_size).cuda() + logits[:, 6] = 100.0 + logits[:, 0] = float(step) + 1.0 + step_logits.append(logits) + sampled = draft_watermarker.sample( + logits, + idx_mapping, + temperature=torch.ones(num_reqs).cuda(), + seed=torch.zeros(num_reqs, dtype=torch.int64).cuda(), + pos=torch.full((num_reqs,), 5 + step, dtype=torch.int64).cuda(), + apply_temperature=True, + is_drafting=True, + logits_cache_col=torch.tensor(step).cuda(), + logits_cache=draft_logits, + use_fp64=False, + ) + assert torch.equal(sampled.cpu(), torch.tensor([6, 6])) + + # Step 0: request 0's [5, 6] is committed at position 2, request 1's [6, 3] + # is new. Step 1: [6, 6] and [3, 6] are both new. Step 2: request 0's + # [6, 6] repeats its own step-1 context and request 1's repeats position 4. + assert [mask.tolist() for mask in skip_masks] == [ + [True, False], + [False, False], + [True, True], + ] + assert torch.equal( + draft_watermarker.prior_contexts[:num_reqs].cpu(), + torch.tensor([[[5, 6], [6, 6], [6, 6]], [[6, 3], [3, 6], [6, 6]]]), + ) + assert torch.equal( + draft_watermarker.contexts[:num_reqs].cpu(), torch.tensor([[6, 6], [6, 6]]) + ) + for step, logits in enumerate(step_logits): + # The 0-d draft step selects the logits cache column. + assert torch.equal(draft_logits[:, step, :vocab_size], logits) + assert not draft_logits[:, :, -1].any() + + def test_dspark_reduced_vocab_draft_sampler_applies_watermarking(monkeypatch): speculator = object.__new__(DSparkSpeculator) speculator.draft_logits = torch.zeros(2, 1, 8) @@ -982,15 +1511,23 @@ def test_dspark_reduced_vocab_draft_sampler_applies_watermarking(monkeypatch): speculator.acceptance_estimator = None watermark_logits: list[torch.Tensor] = [] - def sample(logits, sampled, idx_map, temperature): + def sample( + logits, + idx_mapping, + temperature, + seed, + pos, + apply_temperature, + is_drafting, + logits_cache=None, + logits_cache_col=None, + use_fp64=False, + logits_cache_source=None, + ): watermark_logits.append(logits.clone()) - return sampled + 1 + return torch.tensor([4, 5]) speculator.draft_watermarker = SimpleNamespace(sample=sample) - monkeypatch.setattr( - "vllm.v1.worker.gpu.spec_decode.dspark.speculator.gumbel_sample", - lambda *args, **kwargs: torch.tensor([3, 4]), - ) sampled = speculator._sample_logits( torch.tensor([[10.0, 20.0], [30.0, 40.0]]), diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 824c49efa0..2da17707e5 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -1344,13 +1344,6 @@ class VllmConfig: "share of output tokens supplied by accepted drafts.", scope="global", ) - if watermark_config.deduplicate_contexts != "none": - logger.warning_once( - "Context deduplication is not supported with speculative " - "decoding and will not be applied to accepted drafts, " - "rejection-recovery tokens, or bonus tokens.", - scope="global", - ) if watermark_config.algorithm == "dual_key_gumbel" and ( watermark_config.alpha != get_field(WatermarkConfig, "alpha").default ): diff --git a/vllm/v1/watermarking/gpu_sampler.py b/vllm/v1/watermarking/gpu_sampler.py index a4e6e2dd5b..776dece208 100644 --- a/vllm/v1/watermarking/gpu_sampler.py +++ b/vllm/v1/watermarking/gpu_sampler.py @@ -119,8 +119,9 @@ class GPUWatermarkSampler(Sampler): self, expanded_idx_mapping: torch.Tensor, contexts: torch.Tensor, + expanded_local_pos: torch.Tensor | None = None, ) -> torch.Tensor: - return repeated_context_mask( + repeated = repeated_context_mask( self.req_states.all_token_ids.gpu, expanded_idx_mapping, self.req_states.prompt_len.gpu, @@ -128,8 +129,16 @@ class GPUWatermarkSampler(Sampler): contexts, self.deduplicate_contexts_max_history, include_prompt=self.deduplicate_contexts == "all", - skip_partial_context=self.deduplicate_contexts == "all", + skip_partial_context=( + self.deduplicate_contexts == "all" and expanded_local_pos is None + ), + history_offsets=expanded_local_pos, + local_positions=expanded_local_pos, + num_speculative_steps=( + self.num_speculative_tokens if expanded_local_pos is not None else 0 + ), ) + return repeated def _get_contexts( self, diff --git a/vllm/v1/watermarking/gumbel.py b/vllm/v1/watermarking/gumbel.py index cacb620b5e..655267d1e4 100644 --- a/vllm/v1/watermarking/gumbel.py +++ b/vllm/v1/watermarking/gumbel.py @@ -94,6 +94,10 @@ class GumbelWatermarker(Watermarker): seeds=random_sampler.seeds, positions=random_sampler.positions, use_fp64=random_sampler.use_fp64, + is_drafting=random_sampler.is_drafting, + logits_cache=random_sampler.logits_cache, + logits_cache_col=random_sampler.logits_cache_col, + logits_cache_source=random_sampler.logits_cache_source, ) return WatermarkSample(token_ids, logits) diff --git a/vllm/v1/watermarking/spec_decode.py b/vllm/v1/watermarking/spec_decode.py index c3675a3ab9..36d00313ae 100644 --- a/vllm/v1/watermarking/spec_decode.py +++ b/vllm/v1/watermarking/spec_decode.py @@ -3,14 +3,16 @@ import torch -from vllm.config.watermarking import WatermarkConfig +from vllm.config.watermarking import WatermarkConfig, WatermarkContextScope from vllm.v1.watermarking.factory import create_watermarker from vllm.v1.watermarking.gumbel import GumbelWatermarker from vllm.v1.watermarking.prfs import PhiloxPRF from vllm.v1.watermarking.watermarker import ( + RandomSampler, SupportsSpeculativeDecoding, Watermarker, ) +from vllm.v1.worker.gpu.sample.watermark import draft_watermarking_mask from vllm.v1.worker.gpu.spec_decode.rejection_sampler_utils import rejection_sample @@ -20,8 +22,14 @@ class DraftWatermarker: watermarker: Watermarker, max_num_reqs: int, device: torch.device, + num_speculative_steps: int, + deduplicate_contexts: WatermarkContextScope, + deduplicate_contexts_max_history: int | None, ) -> None: self.watermarker = watermarker + self.num_speculative_steps = num_speculative_steps + self.deduplicate_contexts = deduplicate_contexts + self.deduplicate_contexts_max_history = deduplicate_contexts_max_history self.contexts = torch.zeros( max_num_reqs, watermarker.context_width, @@ -29,30 +37,112 @@ class DraftWatermarker: device=device, ) self.enabled = torch.zeros(max_num_reqs, dtype=torch.bool, device=device) + self.prior_contexts = torch.zeros( + max_num_reqs, + num_speculative_steps, + watermarker.context_width, + dtype=torch.int64, + device=device, + ) + self.all_token_ids: torch.Tensor | None = None + self.prompt_lens: torch.Tensor | None = None + self.total_lens: torch.Tensor | None = None - def prepare(self, contexts: torch.Tensor, enabled: torch.Tensor) -> None: + def prepare( + self, + contexts: torch.Tensor, + enabled: torch.Tensor, + all_token_ids: torch.Tensor, + prompt_lens: torch.Tensor, + total_lens: torch.Tensor, + ) -> None: num_reqs = contexts.shape[0] self.contexts[:num_reqs].copy_(contexts) self.enabled[:num_reqs].copy_(enabled) + self.all_token_ids = all_token_ids + self.prompt_lens = prompt_lens + self.total_lens = total_lens + + def _sampling_state( + self, + logits: torch.Tensor, + idx_mapping: torch.Tensor, + request_temperatures: torch.Tensor, + draft_step: int | torch.Tensor, + contexts: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + num_rows = logits.shape[0] + steps = torch.as_tensor(draft_step, device=logits.device, dtype=torch.int64) + if steps.ndim == 0: + steps = steps.expand(num_rows) + enabled = self.enabled[:num_rows] & (request_temperatures != 0) + if self.deduplicate_contexts == "none": + return steps, enabled + assert self.all_token_ids is not None + assert self.prompt_lens is not None + assert self.total_lens is not None + enabled = draft_watermarking_mask( + self.all_token_ids, + idx_mapping, + self.prompt_lens, + self.total_lens, + self.prior_contexts, + contexts, + steps, + enabled, + self.deduplicate_contexts_max_history, + include_prompt=self.deduplicate_contexts == "all", + ) + return steps, enabled def sample( self, logits: torch.Tensor, - ordinary_sampled: torch.Tensor, idx_mapping: torch.Tensor, temperature: torch.Tensor, + seed: torch.Tensor, + pos: torch.Tensor, + apply_temperature: bool, + is_drafting: bool, + logits_cache: torch.Tensor | None = None, + logits_cache_col: torch.Tensor | None = None, + use_fp64: bool = False, + logits_cache_source: torch.Tensor | None = None, ) -> torch.Tensor: + assert apply_temperature + assert is_drafting + assert logits_cache is not None + assert logits_cache_col is not None + num_rows = logits.shape[0] request_temperatures = temperature[idx_mapping] + contexts = self.contexts[:num_rows] + steps, enabled = self._sampling_state( + logits, idx_mapping, request_temperatures, logits_cache_col, contexts + ) + processed_logits = logits / torch.where( request_temperatures == 0, 1, request_temperatures ).unsqueeze(-1) - watermarked = self.watermarker.sample( + random_sampler = RandomSampler( + expanded_idx_mapping=idx_mapping, + temperatures=temperature, + seeds=seed, + positions=pos, + use_fp64=use_fp64, + is_drafting=True, + logits_cache=logits_cache, + logits_cache_col=steps, + logits_cache_source=( + logits if logits_cache_source is None else logits_cache_source + ), + ) + sampled = self.watermarker.sample( processed_logits, - self.contexts[: logits.shape[0]], + contexts, + random_sampler=random_sampler, + skip_mask=~enabled, ).token_ids - enabled = self.enabled[: logits.shape[0]] & (request_temperatures != 0) - sampled = torch.where(enabled, watermarked, ordinary_sampled) - contexts = self.contexts[: logits.shape[0]] + contexts.copy_(torch.cat((contexts[:, 1:], sampled.unsqueeze(-1)), dim=-1)) return sampled @@ -68,9 +158,19 @@ def create_speculative_draft_watermarker( max_num_reqs: int, device: torch.device, allow_target_only: bool, + num_speculative_steps: int = 1, + deduplicate_contexts: WatermarkContextScope = "none", + deduplicate_contexts_max_history: int | None = 8192, ) -> DraftWatermarker | None: if isinstance(watermarker, SupportsSpeculativeDecoding): - return DraftWatermarker(watermarker.draft_watermarker, max_num_reqs, device) + return DraftWatermarker( + watermarker.draft_watermarker, + max_num_reqs, + device, + num_speculative_steps, + deduplicate_contexts, + deduplicate_contexts_max_history, + ) if allow_target_only: return None raise ValueError( @@ -130,9 +230,10 @@ def watermarked_rejection_sample( contexts: torch.Tensor, watermarking: torch.Tensor, watermarker: Watermarker, + watermarking_skip_mask: torch.Tensor | None = None, use_fp64: bool = False, ) -> tuple[torch.Tensor, torch.Tensor]: - """Use the target key for rejection recovery and bonus tokens.""" + """Use the target key except at contexts selected for ordinary sampling.""" assert contexts.shape == (target_logits.shape[0], watermarker.context_width) return rejection_sample( target_logits, @@ -149,5 +250,6 @@ def watermarked_rejection_sample( use_fp64=use_fp64, contexts=contexts, watermarking=watermarking, + watermarking_skip_mask=watermarking_skip_mask, watermark_key=_resolve_watermark_key(watermarker), ) diff --git a/vllm/v1/watermarking/watermarker.py b/vllm/v1/watermarking/watermarker.py index 12570f4ae5..15917d2520 100644 --- a/vllm/v1/watermarking/watermarker.py +++ b/vllm/v1/watermarking/watermarker.py @@ -23,6 +23,10 @@ class RandomSampler: seeds: torch.Tensor positions: torch.Tensor use_fp64: bool = False + is_drafting: bool = False + logits_cache: torch.Tensor | None = None + logits_cache_col: torch.Tensor | None = None + logits_cache_source: torch.Tensor | None = None def __call__(self, logits: torch.Tensor) -> torch.Tensor: return gumbel_sample( @@ -32,7 +36,10 @@ class RandomSampler: self.seeds, self.positions, apply_temperature=False, - is_drafting=False, + is_drafting=self.is_drafting, + logits_cache=self.logits_cache, + logits_cache_col=self.logits_cache_col, + logits_cache_source=self.logits_cache_source, use_fp64=self.use_fp64, ) diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 3bdeda23dd..fed9e80d0c 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -914,8 +914,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): spec_hidden_states = pre_hc_hidden_states[: hidden_states.shape[0]] # type: ignore[union-attr] if isinstance(self.sampler, GPUWatermarkSampler): self.speculator.prepare_watermarking( - self.sampler._get_contexts(input_batch.idx_mapping), - self.sampler.watermarking.gpu[input_batch.idx_mapping], + self.sampler, input_batch.idx_mapping ) with use_workspace_lane(self._draft_workspace_lane): self.speculator.propose( @@ -2194,8 +2193,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): spec_hidden_states = pre_hc_hidden_states[: draft_hidden_states.size(0)] if isinstance(self.sampler, GPUWatermarkSampler): self.speculator.prepare_watermarking( - self.sampler._get_contexts(input_batch.idx_mapping), - self.sampler.watermarking.gpu[input_batch.idx_mapping], + self.sampler, input_batch.idx_mapping ) with use_workspace_lane(self._draft_workspace_lane): draft_tokens = self.speculator.propose( diff --git a/vllm/v1/worker/gpu/sample/gumbel.py b/vllm/v1/worker/gpu/sample/gumbel.py index e54a458709..a34169b733 100644 --- a/vllm/v1/worker/gpu/sample/gumbel.py +++ b/vllm/v1/worker/gpu/sample/gumbel.py @@ -220,6 +220,8 @@ def gumbel_block_argmax( logits_cache_stride_0, logits_cache_stride_1, logits_cache_col_ptr, + logits_cache_source_ptr, + logits_cache_source_stride, vocab_size, IS_DRAFTING: tl.constexpr, APPLY_TEMPERATURE: tl.constexpr, @@ -240,12 +242,16 @@ def gumbel_block_argmax( col = tl.load(logits_cache_col_ptr + token_idx) else: col = tl.load(logits_cache_col_ptr) + cached_logits = tl.load( + logits_cache_source_ptr + token_idx * logits_cache_source_stride + block, + mask=mask, + ) tl.store( logits_cache_ptr + req_state_idx * logits_cache_stride_0 + col * logits_cache_stride_1 + block, - logits, + cached_logits, mask=mask & is_valid_req, ) @@ -275,6 +281,8 @@ def _gumbel_sample_kernel( logits_cache_stride_0, logits_cache_stride_1, logits_cache_col_ptr, + logits_cache_source_ptr, + logits_cache_source_stride, logits_ptr, logits_stride, expanded_idx_mapping_ptr, @@ -312,6 +320,8 @@ def _gumbel_sample_kernel( logits_cache_stride_0, logits_cache_stride_1, logits_cache_col_ptr, + logits_cache_source_ptr, + logits_cache_source_stride, vocab_size, IS_DRAFTING=IS_DRAFTING, APPLY_TEMPERATURE=APPLY_TEMPERATURE, @@ -334,6 +344,7 @@ def gumbel_sample( logits_cache: torch.Tensor | None = None, # [max_num_reqs, num_cols, vocab_size] logits_cache_col: torch.Tensor | None = None, # scalar or [num_tokens] use_fp64: bool = False, + logits_cache_source: torch.Tensor | None = None, ) -> torch.Tensor: # Enforce contiguity on non-strided input tensors expanded_idx_mapping = expanded_idx_mapping.contiguous() @@ -342,11 +353,26 @@ def gumbel_sample( logits_cache_col = logits_cache_col.contiguous() num_tokens, vocab_size = logits.shape if logits_cache is not None: + if logits_cache_source is None: + logits_cache_source = logits + assert logits_cache_source.shape == logits.shape, ( + "logits cache source must match sampled logits shape" + ) + assert logits_cache_source.device == logits.device, ( + "logits cache source must be on the sampled logits device" + ) + assert logits_cache_source.dtype == logits_cache.dtype, ( + "logits cache source and destination must have the same dtype" + ) assert logits_cache.size(-1) >= vocab_size, ( f"draft logits cache vocab dim ({logits_cache.size(-1)}) is narrower " f"than the sampled logits ({vocab_size}). Cached logits would be " "truncated." ) + elif logits_cache_source is not None: + raise ValueError("logits_cache_source requires logits_cache") + if logits_cache_source is not None and logits_cache_source.stride(-1) != 1: + logits_cache_source = logits_cache_source.contiguous() BLOCK_SIZE = 1024 num_blocks = triton.cdiv(vocab_size, BLOCK_SIZE) local_argmax = logits.new_empty(num_tokens, num_blocks, dtype=torch.int64) @@ -362,6 +388,8 @@ def gumbel_sample( logits_cache.stride(0) if logits_cache is not None else 0, logits_cache.stride(1) if logits_cache is not None else 0, logits_cache_col, + logits_cache_source, + logits_cache_source.stride(0) if logits_cache_source is not None else 0, logits, logits.stride(0), expanded_idx_mapping, diff --git a/vllm/v1/worker/gpu/sample/watermark.py b/vllm/v1/worker/gpu/sample/watermark.py index b70d8ae271..216d189f8d 100644 --- a/vllm/v1/worker/gpu/sample/watermark.py +++ b/vllm/v1/worker/gpu/sample/watermark.py @@ -51,12 +51,25 @@ def _repeated_context_mask_kernel( req_indices_ptr, prompt_lens_ptr, total_lens_ptr, + history_offsets_ptr, + history_offsets_stride, contexts_ptr, context_stride, + local_positions_ptr, + prior_contexts_ptr, + prior_contexts_stride_0, + prior_contexts_stride_1, + steps_ptr, + steps_stride, + enabled_ptr, CONTEXT_WIDTH: tl.constexpr, + NUM_SPECULATIVE_STEPS: tl.constexpr, MAX_HISTORY: tl.constexpr, INCLUDE_PROMPT: tl.constexpr, SKIP_PARTIAL_CONTEXT: tl.constexpr, + HAS_HISTORY_OFFSETS: tl.constexpr, + SPECULATIVE_CONTEXTS: tl.constexpr, + DRAFT_CONTEXTS: tl.constexpr, BLOCK: tl.constexpr, ): row = tl.program_id(0).to(tl.int64) @@ -71,12 +84,16 @@ def _repeated_context_mask_kernel( return sequence_start = 0 if INCLUDE_PROMPT else prompt_len history_len = total_len - sequence_start + history_offset = 0 + if HAS_HISTORY_OFFSETS: + history_offset = tl.load(history_offsets_ptr + row * history_offsets_stride) offsets = tl.arange(0, BLOCK) repeated = tl.full((), 0, tl.int32) scan_start = 0 if MAX_HISTORY > 0: - scan_start = tl.maximum(history_len - MAX_HISTORY, 0) + remaining_history = tl.maximum(MAX_HISTORY - history_offset, 0) + scan_start = tl.maximum(history_len - remaining_history, 0) # Aligned loop bounds let the per-lane offsets fold into immediates. aligned_start = (scan_start // BLOCK) * BLOCK for block_start in tl.range(aligned_start, history_len, BLOCK): @@ -96,6 +113,93 @@ def _repeated_context_mask_kernel( ) matches &= historical_token == context_token repeated |= tl.max(matches.to(tl.int32), axis=0) + + if SPECULATIVE_CONTEXTS: + local_pos = tl.load(local_positions_ptr + row) + if INCLUDE_PROMPT: + partial_context = tl.full((), 0, tl.int32) + for context_offset in range(CONTEXT_WIDTH): + context_token = tl.load( + contexts_ptr + row * context_stride + context_offset + ) + partial_context |= context_token < 0 + repeated |= partial_context + for offset in range(1, NUM_SPECULATIVE_STEPS + 1): + prior_row = row - offset + in_history = (prior_row >= 0) & (local_pos >= offset) + if MAX_HISTORY > 0: + in_history &= offset <= MAX_HISTORY + prior_req_idx = tl.load( + req_indices_ptr + tl.maximum(prior_row, 0), + mask=in_history, + other=-1, + ) + prior_local_pos = tl.load( + local_positions_ptr + tl.maximum(prior_row, 0), + mask=in_history, + other=-1, + ) + matches = ( + in_history + & (req_idx == prior_req_idx) + & (local_pos == prior_local_pos + offset) + ) + for context_offset in range(CONTEXT_WIDTH): + context_token = tl.load( + contexts_ptr + row * context_stride + context_offset + ) + prior_token = tl.load( + contexts_ptr + + tl.maximum(prior_row, 0) * context_stride + + context_offset, + mask=in_history, + other=-2, + ) + matches &= context_token == prior_token + repeated |= matches + + if DRAFT_CONTEXTS: + step = tl.load(steps_ptr + row * steps_stride).to(tl.int64) + partial_context = tl.full((), 0, tl.int32) + for context_offset in range(CONTEXT_WIDTH): + context_token = tl.load( + contexts_ptr + row * context_stride + context_offset + ) + partial_context |= context_token < 0 + if INCLUDE_PROMPT: + repeated |= partial_context + + for prior_step in range(NUM_SPECULATIVE_STEPS): + in_history = prior_step < step + if MAX_HISTORY > 0: + in_history &= prior_step >= step - MAX_HISTORY + matches = in_history + for context_offset in range(CONTEXT_WIDTH): + context_token = tl.load( + contexts_ptr + row * context_stride + context_offset + ) + prior_token = tl.load( + prior_contexts_ptr + + row * prior_contexts_stride_0 + + prior_step * prior_contexts_stride_1 + + context_offset + ) + matches &= context_token == prior_token + repeated |= matches + + for context_offset in range(CONTEXT_WIDTH): + context_token = tl.load( + contexts_ptr + row * context_stride + context_offset + ) + tl.store( + prior_contexts_ptr + + row * prior_contexts_stride_0 + + step * prior_contexts_stride_1 + + context_offset, + context_token, + ) + enabled = tl.load(enabled_ptr + row) + repeated = enabled & (repeated == 0) tl.store(output_ptr + row, repeated) @@ -108,6 +212,7 @@ def _repeated_context_mask_cpu( max_history: int | None = None, include_prompt: bool = False, skip_partial_context: bool = False, + history_offsets: torch.Tensor | None = None, ) -> torch.Tensor: """Reference implementation; the parity tests check the Triton kernel against it.""" repeated = torch.zeros(len(req_indices), dtype=torch.bool) @@ -124,8 +229,11 @@ def _repeated_context_mask_cpu( history_tokens = all_token_ids[req_idx, sequence_start:total_len].tolist() prefix = [-1] * contexts.shape[-1] current_context = tuple(contexts[row].tolist()) + history_offset = 0 if history_offsets is None else int(history_offsets[row]) history_start = ( - max(0, len(history_tokens) - max_history) if max_history is not None else 0 + max(0, len(history_tokens) - max(0, max_history - history_offset)) + if max_history is not None + else 0 ) for history_pos, token_id in enumerate(history_tokens): if history_pos >= history_start and ( @@ -146,6 +254,9 @@ def repeated_context_mask( max_history: int | None = None, include_prompt: bool = False, skip_partial_context: bool = False, + history_offsets: torch.Tensor | None = None, + local_positions: torch.Tensor | None = None, + num_speculative_steps: int = 0, ) -> torch.Tensor: """Return, per row, whether the row's context already occurred in its history. @@ -163,12 +274,23 @@ def repeated_context_mask( include_prompt: Search the prompt as well as the generated tokens. skip_partial_context: Mark contexts containing start padding so they use ordinary sampling. + history_offsets: Number of newer, non-committed positions preceding each + context. These positions count toward `max_history`. Must be 1-D + over rows; a stride-0 broadcast row is fine. + local_positions: Position within each request's speculative block. When + provided, compare each context with earlier contexts in that block. + num_speculative_steps: Maximum number of draft tokens in the block. """ if max_history is not None and max_history < 1: raise ValueError("max_history must be positive or None") + if (local_positions is None) != (num_speculative_steps == 0): + raise ValueError( + "local_positions and positive num_speculative_steps must be " + "provided together" + ) if all_token_ids.device.type == "cpu": - return _repeated_context_mask_cpu( + repeated = _repeated_context_mask_cpu( all_token_ids, req_indices, prompt_lens, @@ -177,10 +299,42 @@ def repeated_context_mask( max_history, include_prompt, skip_partial_context, + history_offsets, ) + if local_positions is not None: + max_offset = num_speculative_steps + 1 + if max_history is not None: + max_offset = min(max_offset, max_history + 1) + for offset in range(1, max_offset): + prior_contexts = torch.cat( + (contexts[:offset], contexts[:-offset]), dim=0 + ) + prior_requests = torch.cat( + (req_indices[:offset], req_indices[:-offset]), dim=0 + ) + prior_local_pos = torch.cat( + (local_positions[:offset], local_positions[:-offset]), dim=0 + ) + repeated |= ( + (req_indices >= 0) + & (local_positions >= offset) + & (req_indices == prior_requests) + & (local_positions == prior_local_pos + offset) + & (contexts == prior_contexts).all(dim=-1) + ) + if include_prompt: + repeated |= (contexts < 0).any(dim=-1) + return repeated if contexts.stride(-1) != 1: contexts = contexts.contiguous() + # Only history_offsets passes its row stride to the kernel. + req_indices = req_indices.contiguous() + if local_positions is not None: + local_positions = local_positions.contiguous() + # `prompt_lens` and `total_lens` are indexed flat by request slot, and + # `all_token_ids` takes a row stride but is read flat within the row. Those + # three are contiguous by contract; every caller owns them outright. repeated = torch.empty(len(req_indices), dtype=torch.bool, device=contexts.device) _repeated_context_mask_kernel[(len(req_indices),)]( repeated, @@ -189,17 +343,104 @@ def repeated_context_mask( req_indices, prompt_lens, total_lens, + history_offsets, + 0 if history_offsets is None else history_offsets.stride(0), contexts, contexts.stride(0), + local_positions, + None, + 0, + 0, + None, + 0, + None, CONTEXT_WIDTH=contexts.shape[-1], + NUM_SPECULATIVE_STEPS=num_speculative_steps, MAX_HISTORY=0 if max_history is None else max_history, INCLUDE_PROMPT=include_prompt, SKIP_PARTIAL_CONTEXT=skip_partial_context, + HAS_HISTORY_OFFSETS=history_offsets is not None, + SPECULATIVE_CONTEXTS=local_positions is not None, + DRAFT_CONTEXTS=False, BLOCK=512, ) return repeated +def draft_watermarking_mask( + all_token_ids: torch.Tensor, + req_indices: torch.Tensor, + prompt_lens: torch.Tensor, + total_lens: torch.Tensor, + prior_contexts: torch.Tensor, + contexts: torch.Tensor, + steps: torch.Tensor, + enabled: torch.Tensor, + max_history: int | None, + include_prompt: bool, +) -> torch.Tensor: + if contexts.device.type == "cpu": + repeated = repeated_context_mask( + all_token_ids, + req_indices, + prompt_lens, + total_lens, + contexts, + max_history, + include_prompt=include_prompt, + history_offsets=steps, + ) + prior_steps = torch.arange(prior_contexts.shape[1]).unsqueeze(0) + in_history = prior_steps < steps.unsqueeze(1) + if max_history is not None: + in_history &= prior_steps >= steps.unsqueeze(1) - max_history + repeated |= ( + (prior_contexts[: len(contexts)] == contexts.unsqueeze(1)).all(dim=-1) + & in_history + ).any(dim=-1) + if include_prompt: + repeated |= (contexts < 0).any(dim=-1) + row_indices = torch.arange(len(contexts)) + prior_contexts.index_put_((row_indices, steps), contexts) + return enabled & ~repeated + + if contexts.stride(-1) != 1: + contexts = contexts.contiguous() + # Only `steps` passes its row stride to the kernel. + req_indices = req_indices.contiguous() + enabled = enabled.contiguous() + active = torch.empty(len(contexts), dtype=torch.bool, device=contexts.device) + _repeated_context_mask_kernel[(len(contexts),)]( + active, + all_token_ids, + all_token_ids.stride(0), + req_indices, + prompt_lens, + total_lens, + steps, + steps.stride(0), + contexts, + contexts.stride(0), + None, + prior_contexts, + prior_contexts.stride(0), + prior_contexts.stride(1), + steps, + steps.stride(0), + enabled, + NUM_SPECULATIVE_STEPS=prior_contexts.shape[1], + CONTEXT_WIDTH=contexts.shape[1], + MAX_HISTORY=0 if max_history is None else max_history, + INCLUDE_PROMPT=include_prompt, + SKIP_PARTIAL_CONTEXT=False, + HAS_HISTORY_OFFSETS=True, + SPECULATIVE_CONTEXTS=False, + DRAFT_CONTEXTS=True, + BLOCK=512, + ) + return active + + @triton.jit def _mulhilo32(multiplier: tl.constexpr, value): high = tl_math.umulhi(multiplier, value) @@ -351,28 +592,58 @@ def _philox_gumbel_kernel( seeds_ptr, pos_ptr, temp_ptr, + logits_cache_ptr, + logits_cache_stride_0, + logits_cache_stride_1, + logits_cache_col_ptr, + logits_cache_source_ptr, + logits_cache_source_stride, key_0_value, key_1_value, vocab_size, CONTEXT_WIDTH: tl.constexpr, BLOCK_SIZE: tl.constexpr, USE_FP64: tl.constexpr, + IS_DRAFTING: tl.constexpr, + PER_TOKEN_COL: tl.constexpr, ): row = tl.program_id(0).to(tl.int64) block_index = tl.program_id(1) groups = block_index * (BLOCK_SIZE // 4) + tl.arange(0, BLOCK_SIZE // 4) + candidate = block_index * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) + mask = candidate < vocab_size - if skip_mask_ptr is not None: + req_state_idx = row + valid_req = True + if expanded_idx_mapping_ptr is not None: req_state_idx = tl.load(expanded_idx_mapping_ptr + row).to(tl.int64) valid_req = req_state_idx >= 0 + + if logits_cache_ptr is not None: + if PER_TOKEN_COL: + col = tl.load(logits_cache_col_ptr + row) + else: + col = tl.load(logits_cache_col_ptr) + cached_logits = tl.load( + logits_cache_source_ptr + row * logits_cache_source_stride + candidate, + mask=mask, + ) + tl.store( + logits_cache_ptr + + req_state_idx * logits_cache_stride_0 + + col * logits_cache_stride_1 + + candidate, + cached_logits, + mask=mask & valid_req, + ) + + if skip_mask_ptr is not None: # noqa: SIM102 if tl.load(skip_mask_ptr + row): temp = tl.load(temp_ptr + req_state_idx, mask=valid_req, other=0.0).to( tl.float32 ) seed = tl.load(seeds_ptr + req_state_idx, mask=valid_req, other=0) pos = tl.load(pos_ptr + row) - candidate = block_index * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) - mask = candidate < vocab_size logits_row = logits_ptr + row * logits_stride logits = tl.load(logits_row + candidate, mask=mask, other=float("-inf")).to( tl.float32 @@ -384,7 +655,7 @@ def _philox_gumbel_kernel( seed, pos, temp, - IS_DRAFTING=False, + IS_DRAFTING=IS_DRAFTING, USE_FP64=USE_FP64, APPLY_TEMPERATURE=False, ) @@ -443,12 +714,37 @@ def philox_gumbel_sample( seeds: torch.Tensor | None = None, positions: torch.Tensor | None = None, use_fp64: bool = False, + is_drafting: bool = False, + logits_cache: torch.Tensor | None = None, + logits_cache_col: torch.Tensor | None = None, + logits_cache_source: torch.Tensor | None = None, ) -> torch.Tensor: sampling_state = (expanded_idx_mapping, temperatures, seeds, positions) if skip_mask is None and any(value is not None for value in sampling_state): raise ValueError("sampling state requires skip_mask") if skip_mask is not None and any(value is None for value in sampling_state): raise ValueError("skip_mask requires complete sampling state") + if logits_cache is not None: + if logits_cache_col is None or logits_cache_source is None: + raise ValueError("logits cache requires its column and source logits") + if expanded_idx_mapping is None: + raise ValueError("logits cache requires expanded_idx_mapping") + assert logits_cache_source.shape == logits.shape, ( + "logits cache source must match sampled logits shape" + ) + assert logits_cache_source.device == logits.device, ( + "logits cache source must be on the sampled logits device" + ) + assert logits_cache_source.dtype == logits_cache.dtype, ( + "logits cache source and destination must have the same dtype" + ) + assert logits_cache.size(-1) >= logits.shape[-1], ( + f"draft logits cache vocab dim ({logits_cache.size(-1)}) is narrower " + f"than the sampled logits ({logits.shape[-1]}). Cached logits would be " + "truncated." + ) + elif logits_cache_col is not None or logits_cache_source is not None: + raise ValueError("logits cache column and source require logits_cache") if logits.stride(-1) != 1: logits = logits.contiguous() @@ -460,6 +756,10 @@ def philox_gumbel_sample( expanded_idx_mapping = expanded_idx_mapping.contiguous() if positions is not None: positions = positions.contiguous() + if logits_cache_col is not None: + logits_cache_col = logits_cache_col.contiguous() + if logits_cache_source is not None and logits_cache_source.stride(-1) != 1: + logits_cache_source = logits_cache_source.contiguous() num_tokens, vocab_size = logits.shape block_size = 1024 num_blocks = triton.cdiv(vocab_size, block_size) @@ -483,12 +783,20 @@ def philox_gumbel_sample( seeds, positions, temperatures, + logits_cache, + logits_cache.stride(0) if logits_cache is not None else 0, + logits_cache.stride(1) if logits_cache is not None else 0, + logits_cache_col, + logits_cache_source, + logits_cache_source.stride(0) if logits_cache_source is not None else 0, key & _UINT32_MASK_VALUE, key >> 32, vocab_size, CONTEXT_WIDTH=contexts.shape[-1], BLOCK_SIZE=block_size, USE_FP64=use_fp64, + IS_DRAFTING=is_drafting, + PER_TOKEN_COL=logits_cache_col is not None and logits_cache_col.dim() > 0, ) max_block_index = local_max.argmax(dim=-1, keepdim=True) return local_argmax.gather(dim=-1, index=max_block_index).view(-1) diff --git a/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py b/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py index b86f58a450..02dd336a6b 100644 --- a/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py @@ -135,7 +135,12 @@ class DSparkSpeculator(DFlashSpeculator): # sample_pos is the predicted token's position P. Sampling keys a draw # by the position before the sampled token, P-1. - sampled = gumbel_sample( + sampler = ( + gumbel_sample + if self.draft_watermarker is None + else self.draft_watermarker.sample + ) + return sampler( logits, idx_map, self.temperature, @@ -147,11 +152,6 @@ class DSparkSpeculator(DFlashSpeculator): logits_cache_col=self._step_cols[step], use_fp64=self.use_fp64_gumbel, ) - if self.draft_watermarker is not None: - sampled = self.draft_watermarker.sample( - logits, sampled, idx_map, self.temperature - ) - return sampled def _sample_sequential(self, num_reqs: int, head_hidden: torch.Tensor) -> None: if self._draft_topk is not None: diff --git a/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py b/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py index 61eb3fb49a..a9ad00e89f 100644 --- a/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py +++ b/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py @@ -226,11 +226,18 @@ class RejectionSampler: if self.watermark_key is None: return {} assert isinstance(self.sampler, GPUWatermarkSampler) + contexts = self.sampler._get_contexts( + expanded_idx_mapping, expanded_local_pos, draft_sampled + ) + watermarking_skip_mask = None + if self.sampler.deduplicate_contexts != "none": + watermarking_skip_mask = self.sampler._get_repeated_contexts( + expanded_idx_mapping, contexts, expanded_local_pos + ) return { - "contexts": self.sampler._get_contexts( - expanded_idx_mapping, expanded_local_pos, draft_sampled - ), + "contexts": contexts, "watermarking": self.sampler.watermarking.gpu, + "watermarking_skip_mask": watermarking_skip_mask, "watermark_key": self.watermark_key, } diff --git a/vllm/v1/worker/gpu/spec_decode/rejection_sampler_utils.py b/vllm/v1/worker/gpu/spec_decode/rejection_sampler_utils.py index cf804e6a6b..b2efa6432e 100644 --- a/vllm/v1/worker/gpu/spec_decode/rejection_sampler_utils.py +++ b/vllm/v1/worker/gpu/spec_decode/rejection_sampler_utils.py @@ -564,7 +564,6 @@ def _rejection_kernel( # stores the target argmax upon first rejection, so it rejects the # placeholder via `accepted` instead. verifying &= is_valid_draft - if verifying: pos = tl.load(pos_ptr + logit_idx) u = tl_rand32(seed, pos, includes_zero=False) @@ -719,6 +718,8 @@ def _seeded_resample_argmax( 0, # logits_cache_stride_0 0, # logits_cache_stride_1 None, # logits_cache_col_ptr + None, # logits_cache_source_ptr + 0, # logits_cache_source_stride vocab_size, IS_DRAFTING=False, APPLY_TEMPERATURE=False, @@ -766,6 +767,7 @@ def _resample_kernel( contexts_stride, # [max_num_reqs], uint8 view of a bool tensor watermarking_ptr, + watermarking_skip_mask_ptr, watermark_key_0, watermark_key_1, vocab_size, @@ -775,6 +777,7 @@ def _resample_kernel( USE_BLOCK_VERIFICATION: tl.constexpr, CONTEXT_WIDTH: tl.constexpr, WATERMARK: tl.constexpr, + DEDUPLICATE_CONTEXTS: tl.constexpr, ): req_idx = tl.program_id(0) resample_idx = tl.load(rejected_step_ptr + req_idx) @@ -807,6 +810,25 @@ def _resample_kernel( ).to(tl.float32) # Compute the residual logits to resample the rejected token from. + is_watermarked = False + if WATERMARK: + is_watermarked = ( + tl.load( + watermarking_ptr + req_state_idx, + mask=req_state_idx >= 0, + other=0, + ) + != 0 + ) + + skip_watermarking = False + if DEDUPLICATE_CONTEXTS: + skip_watermarking = ( + (tl.load(watermarking_skip_mask_ptr + resample_token_idx) != 0) + & is_watermarked + & (temp != 0.0) + ) + if is_bonus or not is_valid_rejected_draft: # Bonus token (no rejections) or -1 placeholder token. In either case, # directly use the target logits. @@ -868,15 +890,7 @@ def _resample_kernel( # Resample the rejected/bonus token. if WATERMARK: # Padded and greedy rows retain the stock draw. - is_watermarked = ( - tl.load( - watermarking_ptr + req_state_idx, - mask=req_state_idx >= 0, - other=0, - ) - != 0 - ) - if is_watermarked & (temp != 0.0): + if is_watermarked & (temp != 0.0) & ~skip_watermarking: watermark_value, idx = philox_gumbel_block_argmax( residual_logits, mask, @@ -1023,6 +1037,7 @@ def rejection_sample( use_block_verification: bool = False, contexts: torch.Tensor | None = None, watermarking: torch.Tensor | None = None, + watermarking_skip_mask: torch.Tensor | None = None, watermark_key: int | None = None, ) -> tuple[torch.Tensor, torch.Tensor]: assert target_logits.ndim == 2 and target_logits.stride(-1) == 1 @@ -1036,9 +1051,13 @@ def rejection_sample( assert watermark == (watermarking is not None) == (watermark_key is not None), ( "contexts, watermarking and watermark_key must be set together." ) + assert watermarking_skip_mask is None or watermark, ( + "watermarking_skip_mask requires watermarking." + ) contexts_stride = 0 context_width = 1 watermarking_bytes: torch.Tensor | None = None + watermarking_skip_mask_bytes: torch.Tensor | None = None watermark_key_0 = 0 watermark_key_1 = 0 if contexts is not None: @@ -1053,6 +1072,12 @@ def rejection_sample( contexts_stride = contexts.stride(0) context_width = contexts.shape[-1] watermarking_bytes = watermarking.view(torch.uint8) + if watermarking_skip_mask is not None: + assert watermarking_skip_mask.shape == (num_logits,) + assert watermarking_skip_mask.dtype == torch.bool + if not watermarking_skip_mask.is_contiguous(): + watermarking_skip_mask = watermarking_skip_mask.contiguous() + watermarking_skip_mask_bytes = watermarking_skip_mask.view(torch.uint8) watermark_key_0 = watermark_key & 0xFFFFFFFF watermark_key_1 = watermark_key >> 32 draft_logits_stride_0 = 0 @@ -1269,6 +1294,7 @@ def rejection_sample( contexts, contexts_stride, watermarking_bytes, + watermarking_skip_mask_bytes, watermark_key_0, watermark_key_1, vocab_size, @@ -1278,6 +1304,7 @@ def rejection_sample( USE_BLOCK_VERIFICATION=use_block_verification, CONTEXT_WIDTH=context_width, WATERMARK=watermark, + DEDUPLICATE_CONTEXTS=watermarking_skip_mask is not None, ) # Insert the resampled tokens into the output sampled. diff --git a/vllm/v1/worker/gpu/spec_decode/speculator.py b/vllm/v1/worker/gpu/spec_decode/speculator.py index c30b653196..17baddf0a8 100644 --- a/vllm/v1/worker/gpu/spec_decode/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/speculator.py @@ -38,6 +38,7 @@ from vllm.v1.worker.gpu.spec_decode.acceptance_estimator import ( from vllm.v1.worker.utils import AttentionGroup if TYPE_CHECKING: + from vllm.v1.watermarking.gpu_sampler import GPUWatermarkSampler from vllm.v1.worker.gpu.pcp_manager import PCPManager logger = init_logger(__name__) @@ -198,6 +199,9 @@ class DraftModelSpeculator(BaseSpeculator): self.max_num_reqs, device, watermark_config.allow_target_only_watermarking, + self.num_speculative_steps, + watermark_config.deduplicate_contexts, + watermark_config.deduplicate_contexts_max_history, ) self.supports_mm_inputs = False @@ -411,7 +415,12 @@ class DraftModelSpeculator(BaseSpeculator): ) -> torch.Tensor: if draft_logits is not None: logits = self.model.compute_logits(hidden_states) - sampled = gumbel_sample( + sampler = ( + gumbel_sample + if self.draft_watermarker is None + else self.draft_watermarker.sample + ) + sampled = sampler( logits, idx_mapping, temperature, @@ -423,10 +432,6 @@ class DraftModelSpeculator(BaseSpeculator): logits_cache_col=draft_step, use_fp64=self.use_fp64_gumbel, ) - if self.draft_watermarker is not None: - sampled = self.draft_watermarker.sample( - logits, sampled, idx_mapping, temperature - ) elif self.use_local_argmax_reduction: return self.model.get_top_tokens(hidden_states) else: @@ -462,11 +467,19 @@ class DraftModelSpeculator(BaseSpeculator): self.acceptance_estimator.step(idx_mapping, num_sampled, num_rejected) def prepare_watermarking( - self, contexts: torch.Tensor, watermarking: torch.Tensor + self, + sampler: "GPUWatermarkSampler", + idx_mapping: torch.Tensor, ) -> None: if self.draft_watermarker is None: return - self.draft_watermarker.prepare(contexts, watermarking) + self.draft_watermarker.prepare( + sampler._get_contexts(idx_mapping), + sampler.watermarking.gpu[idx_mapping], + sampler.req_states.all_token_ids.gpu, + sampler.req_states.prompt_len.gpu, + sampler.req_states.total_len.gpu, + ) def _copy_request_inputs( self,