[watermarking] add context deduplication support to speculative decoding (#56807)

Signed-off-by: Raphael Rialland <raphael.rialland@mistral.ai>
Signed-off-by: Simon Veitner <sveitner@redhat.com>
Co-authored-by: Simon Veitner <sveitner@redhat.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Raphaël Rialland
2026-09-30 12:22:10 -04:00
committed by GitHub
co-authored by Simon Veitner Lucas Wilkinson
parent 73a5831127
commit 0f8b398158
17 changed files with 1379 additions and 88 deletions
+14 -15
View File
@@ -77,30 +77,17 @@ watermarked. The watermark signal is diluted in proportion to the share of
output tokens supplied by accepted drafts; rejected drafts do not dilute it
because their recovery tokens are watermarked.
Speculative-decoding token paths do not currently support generation-side
context deduplication. The configured `deduplicate_contexts` policy is not
applied to accepted drafts, rejection-recovery tokens, or bonus tokens.
For `dual_key_gumbel`, `alpha` has no effect under speculative decoding. The
speculative protocol selects the key for each token instead.
## Algorithms
### Gumbel-max
Gumbel-max derives a deterministic pseudorandom value from the key, prior token
context, and every candidate token, then uses the resulting Gumbel noise for
categorical sampling. See
[Aaronson's original presentation](https://simons.berkeley.edu/sites/default/files/2024-10/LLM24-2%20Slides%20-%20Scott%20Aaronson.pdf).
Gumbel-max requires stochastic sampling. Greedy requests (`temperature=0`)
bypass watermarking and emit a warning once per worker.
## Context deduplication
When a token context is repeated, generation can use ordinary sampling for that
occurrence. Reusing the repeated context leads to a bias over the sequence, as
certain token choices would be correlated. Ordinary sampling at these positions
allows for single-sequence non-distortion (see section G.3 of the
[SynthID-Text supplementary materials](https://media.springernature.com/original/springer-static/esm/art%3A10.1038%2Fs41586-024-08025-4/MediaObjects/41586_2024_8025_MOESM1_ESM.pdf)).
This also applies to drafter watermarking during speculative decoding.
The detector independently deduplicates contexts so repeated keyed random
vectors are not treated as independent evidence, meaning that context
deduplication at generation time does not reduce the watermarking signal, unless
@@ -143,6 +130,18 @@ vllm serve MODEL \
'{"algorithm":"gumbel","key":42,"deduplicate_contexts":"all"}'
```
## Algorithms
### Gumbel-max
Gumbel-max derives a deterministic pseudorandom value from the key, prior token
context, and every candidate token, then uses the resulting Gumbel noise for
categorical sampling. See
[Aaronson's original presentation](https://simons.berkeley.edu/sites/default/files/2024-10/LLM24-2%20Slides%20-%20Scott%20Aaronson.pdf).
Gumbel-max requires stochastic sampling. Greedy requests (`temperature=0`)
bypass watermarking and emit a warning once per worker.
### Dual-key Gumbel-max
Dual-key Gumbel-max derives independent keys A and B from one configured master
+23 -2
View File
@@ -3526,7 +3526,6 @@ def test_target_only_gumbel_allows_speculative_decoding(caplog_vllm, disable_log
config._check_watermarking_unsupported()
assert "Target-only watermarking leaves accepted draft tokens" in caplog_vllm.text
assert "Context deduplication is not supported" in caplog_vllm.text
def test_speculative_watermarking_without_context_dedup_does_not_warn(
@@ -3543,10 +3542,32 @@ def test_speculative_watermarking_without_context_dedup_does_not_warn(
parallel_drafting=False,
)
caplog_vllm.clear()
with caplog_vllm.at_level(logging.WARNING):
config._check_watermarking_unsupported()
assert "Context deduplication is not supported" not in caplog_vllm.text
assert "dedup" not in caplog_vllm.text.lower()
def test_speculative_context_dedup_logs_no_unsupported_warning(
caplog_vllm, disable_log_dedup
):
config = _watermarked_vllm_config()
config.watermark_config = WatermarkConfig(
algorithm="dual_key_gumbel", key=42, deduplicate_contexts="single_turn"
)
config.speculative_config = SimpleNamespace(
method="mtp",
draft_sample_method="probabilistic",
rejection_sample_method="standard",
parallel_drafting=False,
)
caplog_vllm.clear()
with caplog_vllm.at_level(logging.WARNING):
config._check_watermarking_unsupported()
assert "dedup" not in caplog_vllm.text.lower()
def test_gumbel_rejects_speculative_decoding_without_target_only():
@@ -203,6 +203,182 @@ def test_watermarked_recovery_uses_target_key():
assert sampled[0, 0] == expected.token_ids[0]
def test_repeated_draft_context_does_not_change_acceptance():
torch.manual_seed(0)
num_trials = 8
# A draft distribution of its own, so the drafts are not accepted outright.
target_logits = torch.randn(64, device="cuda")
draft_logits = torch.randn(64, device="cuda")
inputs = _build_rejection_sample_inputs(
target_logits,
draft_logits,
num_speculative_steps=1,
temperature=1.0,
num_trials=num_trials,
)
expected, expected_num_sampled = rejection_sample(**inputs, num_speculative_steps=1)
sampled, num_sampled = watermarked_rejection_sample(
**inputs,
num_speculative_steps=1,
contexts=torch.tensor([[10, 11], [11, 12]], device="cuda").repeat(
num_trials, 1
),
watermarking=torch.ones(num_trials, dtype=torch.bool, device="cuda"),
watermarking_skip_mask=torch.tensor([True, False], device="cuda").repeat(
num_trials
),
watermarker=GumbelWatermarker(key=42, context_width=2),
)
# Some drafts must actually be rejected, or the ordinary-residual branch
# the skip mask selects is never reached.
assert (expected_num_sampled == 1).any()
torch.testing.assert_close(num_sampled, expected_num_sampled, rtol=0, atol=0)
torch.testing.assert_close(sampled[:, :1], expected[:, :1], rtol=0, atol=0)
def test_repeated_context_skip_mask_is_read_per_row_in_a_block():
"""A block of three drafts, accepted at row 0 and rejected at row 1.
The skip mask selects the sampler per row, so the row the target recovers
at decides which key draws the recovery token.
"""
num_speculative_steps = 3
target_logits = torch.tensor([float("-inf"), 0.0, 1.0, 2.0], device="cuda")
draft_logits = torch.tensor([30.0, -30.0, -30.0, -30.0], device="cuda")
inputs = _build_rejection_sample_inputs(
target_logits,
draft_logits,
num_speculative_steps=num_speculative_steps,
temperature=1.0,
num_trials=1,
)
# Row 0 drafts a token the target gives 0.67 and the draft e**-60, so it is
# accepted; row 1 drafts a token the target gives no mass, so it is not.
inputs["draft_sampled"][1] = 3
inputs["draft_sampled"][2] = 0
contexts = torch.tensor(
[[10, 11], [11, 3], [3, 0], [0, 1]], dtype=torch.int64, device="cuda"
)
expected, expected_num_sampled = rejection_sample(
**inputs, num_speculative_steps=num_speculative_steps
)
assert expected_num_sampled.item() == 2
assert expected[0, 0].item() == 3
watermarker = GumbelWatermarker(key=42, context_width=2)
skipped, skipped_num_sampled = watermarked_rejection_sample(
**inputs,
num_speculative_steps=num_speculative_steps,
contexts=contexts,
watermarking=torch.tensor([True], device="cuda"),
watermarking_skip_mask=torch.tensor([False, True, False, False], device="cuda"),
watermarker=watermarker,
)
assert skipped_num_sampled.item() == 2
torch.testing.assert_close(skipped[:, :2], expected[:, :2], rtol=0, atol=0)
watermarked, watermarked_num_sampled = watermarked_rejection_sample(
**inputs,
num_speculative_steps=num_speculative_steps,
contexts=contexts,
watermarking=torch.tensor([True], device="cuda"),
watermarking_skip_mask=torch.zeros(
num_speculative_steps + 1, dtype=torch.bool, device="cuda"
),
watermarker=watermarker,
)
# The residual equals the target to within e**-60, so the recovery token is
# the target key's draw on the rejected row's context.
recovery = GumbelWatermarker(key=42, context_width=2).sample(
target_logits.unsqueeze(0), contexts[1:2], lambda _: None
)
assert watermarked_num_sampled.item() == 2
assert watermarked[0, 0].item() == 3
assert watermarked[0, 1] == recovery.token_ids[0]
def test_repeated_recovery_context_uses_ordinary_residual_sample():
target_logits = torch.tensor([float("-inf"), 0.0, 1.0, 2.0], device="cuda")
draft_logits = torch.tensor([100.0, -100.0, -100.0, -100.0], device="cuda")
inputs = _build_rejection_sample_inputs(
target_logits,
draft_logits,
num_speculative_steps=1,
temperature=1.0,
num_trials=1,
)
inputs["draft_sampled"][1] = 0
expected, expected_num_sampled = rejection_sample(**inputs, num_speculative_steps=1)
sampled, num_sampled = watermarked_rejection_sample(
**inputs,
num_speculative_steps=1,
contexts=torch.tensor([[10, 11], [11, 0]], device="cuda"),
watermarking=torch.tensor([True], device="cuda"),
watermarking_skip_mask=torch.tensor([True, False], device="cuda"),
watermarker=GumbelWatermarker(key=42, context_width=2),
)
torch.testing.assert_close(num_sampled, expected_num_sampled, rtol=0, atol=0)
torch.testing.assert_close(sampled[:, :1], expected[:, :1], rtol=0, atol=0)
def test_repeated_bonus_context_uses_ordinary_target_sample():
target_logits = torch.randn(64, device="cuda")
inputs = _build_rejection_sample_inputs(
target_logits,
target_logits,
num_speculative_steps=1,
temperature=1.0,
num_trials=1,
)
expected, expected_num_sampled = rejection_sample(**inputs, num_speculative_steps=1)
sampled, num_sampled = watermarked_rejection_sample(
**inputs,
num_speculative_steps=1,
contexts=torch.tensor([[10, 11], [11, 12]], device="cuda"),
watermarking=torch.tensor([True], device="cuda"),
watermarking_skip_mask=torch.tensor([False, True], device="cuda"),
watermarker=GumbelWatermarker(key=42, context_width=2),
)
torch.testing.assert_close(num_sampled, expected_num_sampled, rtol=0, atol=0)
torch.testing.assert_close(sampled[:, :2], expected[:, :2], rtol=0, atol=0)
def test_repeated_context_does_not_change_unwatermarked_or_greedy_rows():
target_logits = torch.randn(64, device="cuda")
inputs = _build_rejection_sample_inputs(
target_logits,
target_logits,
num_speculative_steps=1,
temperature=1.0,
num_trials=2,
)
inputs["temperature"][1] = 0
expected, expected_num_sampled = rejection_sample(**inputs, num_speculative_steps=1)
sampled, num_sampled = watermarked_rejection_sample(
**inputs,
num_speculative_steps=1,
contexts=torch.tensor([[10, 11], [11, 12], [20, 21], [21, 22]], device="cuda"),
watermarking=torch.tensor([False, True], device="cuda"),
watermarking_skip_mask=torch.ones(4, dtype=torch.bool, device="cuda"),
watermarker=GumbelWatermarker(key=42, context_width=2),
)
torch.testing.assert_close(num_sampled, expected_num_sampled, rtol=0, atol=0)
for row, length in enumerate(num_sampled):
torch.testing.assert_close(
sampled[row, : int(length)], expected[row, : int(length)], rtol=0, atol=0
)
def test_watermarked_recovery_supports_smaller_draft_vocabulary():
target_logits = torch.tensor([float("-inf"), 0.0, 1.0, 2.0, 100.0], device="cuda")
draft_logits = torch.tensor([100.0, -100.0, -100.0, -100.0], device="cuda")
+62
View File
@@ -12,6 +12,7 @@ from vllm.v1.watermarking import (
derive_watermark_key,
)
from vllm.v1.watermarking.gumbel import _gamma_survival_integer_shape
from vllm.v1.watermarking.watermarker import RandomSampler
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
from vllm.v1.worker.gpu.sample.watermark import philox_gumbel_sample
@@ -197,3 +198,64 @@ def test_philox_gumbel_sample_skip_mask_matches_separate_samplers(use_fp64: bool
)
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
def test_philox_mixed_draft_sampling_caches_unprocessed_logits():
torch.manual_seed(0)
num_rows, vocab_size, num_steps = 5, 4099, 3
logits = torch.randn(num_rows, vocab_size, device="cuda", dtype=torch.float16)
temperatures = torch.tensor([0.0, 0.3, 0.5, 1.0, 2.0], device="cuda")
processed_logits = logits / torch.where(
temperatures == 0, 1, temperatures
).unsqueeze(-1)
contexts = torch.randint(0, vocab_size, (num_rows, 4), device="cuda")
idx_mapping = torch.arange(num_rows, dtype=torch.int32, device="cuda")
seeds = torch.arange(num_rows, dtype=torch.int64, device="cuda") + 10
positions = torch.arange(num_rows, dtype=torch.int64, device="cuda") + 20
# The skipped rows must divide by a temperature other than 1, or the
# unprocessed logits the cache stores are the logits the sampler sees.
# Row 0 keeps a greedy row in the skipped set.
skip_mask = torch.tensor([True, True, False, False, True], device="cuda")
cols = torch.arange(num_rows, dtype=torch.int32, device="cuda") % num_steps
cache = torch.zeros(
num_rows, num_steps, vocab_size + 1, dtype=logits.dtype, device="cuda"
)
ordinary = gumbel_sample(
logits,
idx_mapping,
temperatures,
seeds,
positions,
apply_temperature=True,
is_drafting=True,
)
watermarked = philox_gumbel_sample(processed_logits, contexts, 42)
expected = torch.where(skip_mask, ordinary, watermarked)
actual = (
GumbelWatermarker(key=42, context_width=4)
.sample(
processed_logits,
contexts,
random_sampler=RandomSampler(
expanded_idx_mapping=idx_mapping,
temperatures=temperatures,
seeds=seeds,
positions=positions,
is_drafting=True,
logits_cache=cache,
logits_cache_col=cols,
logits_cache_source=logits,
),
skip_mask=skip_mask,
)
.token_ids
)
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
stored = cache[torch.arange(num_rows, device="cuda"), cols.long(), :vocab_size]
assert torch.equal(stored.view(torch.int16), logits.view(torch.int16))
assert not cache[:, :, -1].any()
+550 -13
View File
@@ -28,6 +28,7 @@ from vllm.v1.watermarking.spec_decode import (
from vllm.v1.watermarking.watermarker import Watermarker, WatermarkSample
from vllm.v1.worker.gpu.sample.sampler import Sampler
from vllm.v1.worker.gpu.sample.watermark import (
draft_watermarking_mask,
philox_gumbel_sample,
repeated_context_mask,
)
@@ -202,11 +203,17 @@ def test_speculative_decoding_uses_fixed_dual_key_roles():
max_num_reqs=1,
device=torch.device("cpu"),
allow_target_only=False,
num_speculative_steps=3,
deduplicate_contexts="all",
deduplicate_contexts_max_history=None,
)
assert target.prf.key == derive_watermark_key(42, b"key_b")
assert draft is not None
assert draft.watermarker.prf.key == derive_watermark_key(42, b"key_a")
assert draft.num_speculative_steps == 3
assert draft.deduplicate_contexts == "all"
assert draft.deduplicate_contexts_max_history is None
def test_recovery_key_rejects_the_unsplit_dual_key_watermarker():
@@ -892,6 +899,187 @@ def test_gpu_sampler_builds_speculative_contexts_from_drafts():
assert torch.equal(contexts, torch.tensor([[10, 11], [11, 20], [20, 21]]))
def test_gpu_sampler_deduplicates_contexts_within_speculative_block():
sampler = object.__new__(GPUWatermarkSampler)
sampler.watermarker = SimpleNamespace(context_width=2)
sampler.deduplicate_contexts = "single_turn"
sampler.deduplicate_contexts_max_history = None
sampler.num_speculative_tokens = 4
sampler.req_states = SimpleNamespace(
all_token_ids=SimpleNamespace(gpu=torch.tensor([[100, 1, 2, 0, 0, 0]])),
prompt_len=SimpleNamespace(gpu=torch.tensor([1])),
total_len=SimpleNamespace(gpu=torch.tensor([3])),
)
request_indices = torch.tensor([0, 0, 0, 0, 0])
local_positions = torch.tensor([0, 1, 2, 3, 4])
contexts = sampler._get_contexts(
request_indices,
local_positions,
torch.tensor([0, 3, 4, 1, 2]),
)
repeated = sampler._get_repeated_contexts(
request_indices, contexts, local_positions
)
assert torch.equal(
contexts,
torch.tensor([[1, 2], [2, 3], [3, 4], [4, 1], [1, 2]]),
)
assert torch.equal(repeated, torch.tensor([False, False, False, False, True]))
sampler.deduplicate_contexts_max_history = 3
repeated = sampler._get_repeated_contexts(
request_indices, contexts, local_positions
)
assert not repeated.any()
def test_gpu_sampler_all_scope_skips_only_partial_speculative_contexts():
sampler = object.__new__(GPUWatermarkSampler)
sampler.watermarker = SimpleNamespace(context_width=2)
sampler.deduplicate_contexts = "all"
sampler.deduplicate_contexts_max_history = None
sampler.num_speculative_tokens = 3
sampler.req_states = SimpleNamespace(
all_token_ids=SimpleNamespace(gpu=torch.tensor([[100, 101, 10, 0, 0]])),
prompt_len=SimpleNamespace(gpu=torch.tensor([2])),
total_len=SimpleNamespace(gpu=torch.tensor([3])),
)
request_indices = torch.tensor([0, 0, 0])
local_positions = torch.tensor([0, 1, 2])
contexts = sampler._get_contexts(
request_indices,
local_positions,
torch.tensor([10, 20, 30]),
)
repeated = sampler._get_repeated_contexts(
request_indices, contexts, local_positions
)
assert torch.equal(contexts, torch.tensor([[-1, 10], [10, 20], [20, 30]]))
assert torch.equal(repeated, torch.tensor([True, False, False]))
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
@pytest.mark.parametrize("max_history,expected_last", [(None, True), (3, False)])
def test_speculative_context_repetition_accelerator_parity(
max_history: int | None, expected_last: bool
):
all_token_ids = torch.tensor([[10, 11]])
req_indices = torch.tensor([0, 0, 0, 0, 0])
prompt_lens = torch.tensor([0])
total_lens = torch.tensor([2])
local_positions = torch.tensor([0, 1, 2, 3, 4])
contexts = torch.tensor([[1, 2], [2, 3], [3, 4], [4, 1], [1, 2]])
expected = repeated_context_mask(
all_token_ids,
req_indices,
prompt_lens,
total_lens,
contexts,
history_offsets=local_positions,
local_positions=local_positions,
num_speculative_steps=4,
max_history=max_history,
)
actual = repeated_context_mask(
all_token_ids.cuda(),
req_indices.cuda(),
prompt_lens.cuda(),
total_lens.cuda(),
contexts.cuda(),
history_offsets=local_positions.cuda(),
local_positions=local_positions.cuda(),
num_speculative_steps=4,
max_history=max_history,
).cpu()
assert torch.equal(
expected, torch.tensor([False, False, False, False, expected_last])
)
assert torch.equal(actual, expected)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
@pytest.mark.parametrize("steps_layout", ["per_row", "broadcast"])
@pytest.mark.parametrize("req_indices_layout", ["contiguous", "column_view"])
def test_draft_context_deduplication_accelerator_parity(
steps_layout, req_indices_layout
):
"""Parity with the CPU reference for the layouts the speculators pass.
A draft step is one 0-d tensor broadcast over the batch, and DSpark samples
a step at a time, so its request indices are an `idx_map[:, i]` column. Both
are read per row, so a row that ignores the layout consults another row's
step or another request's history.
"""
num_rows, num_steps, step = 4, 3, 1
all_token_ids = torch.tensor([[10, 11, 20, 11], [30, 31, 11, 20]])
prompt_lens = torch.tensor([0, 0])
total_lens = torch.tensor([4, 4])
contexts = torch.tensor([[40, 41], [31, 11], [10, 11], [50, 51]])
prior_contexts = torch.zeros(num_rows, num_steps, 2, dtype=torch.int64)
prior_contexts[0, 0] = torch.tensor([40, 41])
enabled = torch.ones(num_rows, dtype=torch.bool)
req_indices = torch.tensor([0, 1, 0, 1])
def run(all_token_ids, req_indices, steps, prior):
return draft_watermarking_mask(
all_token_ids,
req_indices,
prompt_lens.to(all_token_ids.device),
total_lens.to(all_token_ids.device),
prior,
contexts.to(all_token_ids.device),
steps,
enabled.to(all_token_ids.device),
max_history=3,
include_prompt=False,
)
reference_prior = prior_contexts.clone()
reference = run(
all_token_ids,
req_indices,
torch.full((num_rows,), step),
reference_prior,
)
# Row 0 repeats its own step-0 context, rows 1 and 2 repeat a committed
# context of their own request, row 3 repeats nothing.
assert torch.equal(reference, torch.tensor([False, False, False, True]))
expected_prior = prior_contexts.clone()
expected_prior[:, step] = contexts
assert torch.equal(reference_prior, expected_prior)
if steps_layout == "per_row":
steps = torch.full((num_rows,), step).cuda()
else:
steps = torch.tensor(step).cuda().expand(num_rows)
assert steps.stride(0) == 0
if req_indices_layout == "contiguous":
gpu_req_indices = req_indices.cuda()
else:
gpu_req_indices = (
req_indices.repeat_interleave(num_steps).view(num_rows, num_steps).cuda()
)[:, 1]
assert gpu_req_indices.stride(0) == num_steps
actual_prior = prior_contexts.cuda()
actual = run(all_token_ids.cuda(), gpu_req_indices, steps, actual_prior).cpu()
assert torch.equal(actual, reference)
assert torch.equal(actual_prior.cpu(), expected_prior)
def test_gpu_sampler_builds_chunked_multi_request_speculative_contexts():
sampler = object.__new__(GPUWatermarkSampler)
sampler.watermarker = SimpleNamespace(context_width=2)
@@ -921,6 +1109,69 @@ def test_gpu_sampler_builds_chunked_multi_request_speculative_contexts():
)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
def test_draft_context_deduplication_broadcasts_the_draft_step():
"""The draft step is one 0-d tensor broadcast over the batch.
The context occurs four positions back, which `max_history - step` puts
just outside the scanned window, so a row that mistakes another row's step
for its own reports the opposite verdict. The neighbours leg is the
deterministic control: a row that reads a step of 0 instead of 2 widens its
window by two positions and finds the context. The production leg asserts
the layout the speculators pass, where the stride-less read lands outside
the allocation and can return the right value by chance.
"""
num_rows = 4
all_token_ids = torch.tensor([[3, 4, 9, 9, 9, 9], [3, 4, 9, 9, 9, 9]])
req_indices = torch.tensor([0, 0, 1, 1])
prompt_lens = torch.tensor([0, 0])
total_lens = torch.tensor([6, 6])
prior_contexts = torch.zeros(num_rows, 3, 2, dtype=torch.int64)
contexts = torch.tensor([[3, 4]]).expand(num_rows, 2).contiguous()
enabled = torch.ones(num_rows, dtype=torch.bool)
def run(steps, device):
prior = prior_contexts.clone().to(device)
active = draft_watermarking_mask(
all_token_ids.to(device),
req_indices.to(device),
prompt_lens.to(device),
total_lens.to(device),
prior,
contexts.to(device),
steps,
enabled.to(device),
max_history=4,
include_prompt=False,
)
return active.cpu(), prior.cpu()
expected, expected_prior = run(torch.full((num_rows,), 2), torch.device("cpu"))
assert torch.equal(expected, torch.ones(num_rows, dtype=torch.bool))
materialized, materialized_prior = run(
torch.full((num_rows,), 2).cuda(), torch.device("cuda")
)
assert torch.equal(materialized, expected)
assert torch.equal(materialized_prior, expected_prior)
# Neighbouring steps are the value a row reads when the broadcast stride is
# ignored; the production tensor reads uninitialized memory instead.
neighbours = torch.tensor([2, 0, 0, 0]).cuda()
broadcast, broadcast_prior = run(
neighbours[0].expand(num_rows), torch.device("cuda")
)
assert torch.equal(broadcast, expected)
assert torch.equal(broadcast_prior, expected_prior)
step = torch.tensor(2).cuda()
production, production_prior = run(step.expand(num_rows), torch.device("cuda"))
assert torch.equal(production, expected)
assert torch.equal(production_prior, expected_prior)
def test_draft_sampler_uses_draft_key_and_advances_context(monkeypatch):
class StubSpeculator(DraftModelSpeculator):
def capture(self): ...
@@ -937,22 +1188,32 @@ def test_draft_sampler_uses_draft_key_and_advances_context(monkeypatch):
return torch.zeros(hidden_states.shape[0], 8)
class StubWatermarker:
context_width = 2
@staticmethod
def sample(logits, contexts, random_sampler=None):
assert random_sampler is None
return WatermarkSample(torch.tensor([7, 7]), logits)
def sample(logits, contexts, random_sampler=None, skip_mask=None):
assert random_sampler is not None
return WatermarkSample(
torch.where(skip_mask, random_sampler(logits), 7), logits
)
speculator = object.__new__(StubSpeculator)
speculator.model = StubModel()
speculator.use_fp64_gumbel = False
speculator.acceptance_estimator = None
draft_watermarker = object.__new__(DraftWatermarker)
draft_watermarker.watermarker = StubWatermarker()
draft_watermarker.contexts = torch.tensor([[1, 2], [3, 4]])
draft_watermarker = DraftWatermarker(
StubWatermarker(),
max_num_reqs=2,
device=torch.device("cpu"),
num_speculative_steps=1,
deduplicate_contexts="none",
deduplicate_contexts_max_history=None,
)
draft_watermarker.contexts.copy_(torch.tensor([[1, 2], [3, 4]]))
draft_watermarker.enabled = torch.tensor([True, False])
speculator.draft_watermarker = draft_watermarker
monkeypatch.setattr(
"vllm.v1.worker.gpu.spec_decode.speculator.gumbel_sample",
"vllm.v1.watermarking.watermarker.gumbel_sample",
lambda *args, **kwargs: torch.tensor([3, 4]),
)
@@ -970,6 +1231,274 @@ def test_draft_sampler_uses_draft_key_and_advances_context(monkeypatch):
assert torch.equal(draft_watermarker.contexts, torch.tensor([[2, 7], [4, 4]]))
def test_draft_sampler_uses_ordinary_samples_for_repeated_contexts(monkeypatch):
class StubWatermarker:
context_width = 1
def __init__(self):
self.samples = iter((2, 1, 8))
def sample(self, logits, contexts, random_sampler=None, skip_mask=None):
watermarked = torch.tensor([next(self.samples)])
return WatermarkSample(
torch.where(skip_mask, random_sampler(logits), watermarked), logits
)
draft_watermarker = DraftWatermarker(
StubWatermarker(),
max_num_reqs=1,
device=torch.device("cpu"),
num_speculative_steps=3,
deduplicate_contexts="single_turn",
deduplicate_contexts_max_history=None,
)
draft_watermarker.prepare(
contexts=torch.tensor([[1]]),
enabled=torch.tensor([True]),
all_token_ids=torch.tensor([[100, 1, 0, 0]]),
prompt_lens=torch.tensor([1]),
total_lens=torch.tensor([2]),
)
idx_mapping = torch.tensor([0])
temperature = torch.tensor([1.0])
ordinary_samples = iter((torch.tensor([5]), torch.tensor([6]), torch.tensor([4])))
monkeypatch.setattr(
"vllm.v1.watermarking.watermarker.gumbel_sample",
lambda *args, **kwargs: next(ordinary_samples),
)
seeds = torch.tensor([0])
draft_logits = torch.zeros(1, 3, 8)
first = draft_watermarker.sample(
torch.zeros(1, 8),
idx_mapping,
temperature=temperature,
seed=seeds,
pos=torch.tensor([0]),
apply_temperature=True,
is_drafting=True,
logits_cache_col=torch.tensor(0),
logits_cache=draft_logits,
use_fp64=False,
)
second = draft_watermarker.sample(
torch.zeros(1, 8),
idx_mapping,
temperature=temperature,
seed=seeds,
pos=torch.tensor([1]),
apply_temperature=True,
is_drafting=True,
logits_cache_col=torch.tensor(1),
logits_cache=draft_logits,
use_fp64=False,
)
repeated = draft_watermarker.sample(
torch.zeros(1, 8),
idx_mapping,
temperature=temperature,
seed=seeds,
pos=torch.tensor([2]),
apply_temperature=True,
is_drafting=True,
logits_cache_col=torch.tensor(2),
logits_cache=draft_logits,
use_fp64=False,
)
assert first.item() == 2
assert second.item() == 1
assert repeated.item() == 4
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
def test_draft_context_deduplication_reads_strided_request_indices():
"""DSpark samples a step at a time, so req_indices is an `idx_map[:, i]` column."""
num_reqs = 4
num_speculative_steps = 3
all_token_ids = torch.tensor(
[[3, 4, 3, 4], [9, 9, 9, 9], [3, 4, 3, 4], [9, 9, 9, 9]]
).cuda()
prompt_lens = torch.zeros(num_reqs, dtype=torch.int64).cuda()
total_lens = torch.full((num_reqs,), 4).cuda()
contexts = torch.tensor([[3, 4]]).expand(num_reqs, 2).contiguous().cuda()
steps = torch.zeros(num_reqs, dtype=torch.int64).cuda()
enabled = torch.ones(num_reqs, dtype=torch.bool).cuda()
idx_map = (
torch.arange(num_reqs)
.repeat_interleave(num_speculative_steps)
.view(num_reqs, num_speculative_steps)
.cuda()
)
column = idx_map[:, 1]
assert column.stride(0) == num_speculative_steps
def run(req_indices):
prior = torch.zeros(
num_reqs, num_speculative_steps, 2, dtype=torch.int64, device="cuda"
)
return draft_watermarking_mask(
all_token_ids,
req_indices,
prompt_lens,
total_lens,
prior,
contexts,
steps,
enabled,
max_history=None,
include_prompt=False,
).cpu()
expected = torch.tensor([False, True, False, True])
assert torch.equal(run(column.contiguous()), expected)
assert torch.equal(run(column), expected)
def test_draft_sampler_deduplicates_against_committed_history(monkeypatch):
class StubWatermarker:
context_width = 1
@staticmethod
def sample(logits, contexts, random_sampler=None, skip_mask=None):
return WatermarkSample(
torch.where(skip_mask, random_sampler(logits), 7), logits
)
draft_watermarker = DraftWatermarker(
StubWatermarker(),
max_num_reqs=1,
device=torch.device("cpu"),
num_speculative_steps=1,
deduplicate_contexts="single_turn",
deduplicate_contexts_max_history=None,
)
draft_watermarker.prepare(
contexts=torch.tensor([[1]]),
enabled=torch.tensor([True]),
all_token_ids=torch.tensor([[100, 1, 2, 1]]),
prompt_lens=torch.tensor([1]),
total_lens=torch.tensor([4]),
)
monkeypatch.setattr(
"vllm.v1.watermarking.watermarker.gumbel_sample",
lambda *args, **kwargs: torch.tensor([4]),
)
sampled = draft_watermarker.sample(
torch.zeros(1, 8),
torch.tensor([0]),
temperature=torch.tensor([1.0]),
seed=torch.tensor([0]),
pos=torch.tensor([0]),
apply_temperature=True,
is_drafting=True,
logits_cache_col=torch.tensor(0),
logits_cache=torch.zeros(1, 1, 8),
use_fp64=False,
)
assert sampled.item() == 4
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
@pytest.mark.parametrize("idx_mapping_layout", ["contiguous", "column_view"])
def test_draft_watermarker_deduplicates_across_a_block_on_cuda(
monkeypatch, idx_mapping_layout
):
"""Drive the whole draft sampler over a block with the production shapes.
`draft_step` is a 0-d tensor, and a speculator that samples one step at a
time hands over a column of its index map. Each step is checked against the
committed history the window still reaches and against the earlier steps of
the block.
"""
num_reqs, num_steps, vocab_size = 2, 3, 16
watermarker = GumbelWatermarker(key=42, context_width=2)
draft_watermarker = DraftWatermarker(
watermarker,
max_num_reqs=num_reqs,
device=torch.device("cuda"),
num_speculative_steps=num_steps,
deduplicate_contexts="single_turn",
deduplicate_contexts_max_history=4,
)
draft_watermarker.prepare(
contexts=torch.tensor([[5, 6], [6, 3]]).cuda(),
enabled=torch.ones(num_reqs, dtype=torch.bool).cuda(),
all_token_ids=torch.tensor([[5, 6, 7, 5, 6], [8, 9, 6, 6, 3]]).cuda(),
prompt_lens=torch.zeros(num_reqs, dtype=torch.int64).cuda(),
total_lens=torch.full((num_reqs,), 5).cuda(),
)
if idx_mapping_layout == "contiguous":
idx_mapping = torch.arange(num_reqs, dtype=torch.int32).cuda()
else:
idx_mapping = (
torch.arange(num_reqs, dtype=torch.int32)
.repeat_interleave(num_steps)
.view(num_reqs, num_steps)
.cuda()
)[:, 1]
assert idx_mapping.stride(0) == num_steps
skip_masks: list[torch.Tensor] = []
original = philox_gumbel_sample
def spy(*args, **kwargs):
skip_masks.append(kwargs["skip_mask"].cpu())
return original(*args, **kwargs)
monkeypatch.setattr("vllm.v1.worker.gpu.sample.watermark.philox_gumbel_sample", spy)
draft_logits = torch.zeros(num_reqs, num_steps, vocab_size + 1).cuda()
step_logits = []
for step in range(num_steps):
# Token 6 wins by a margin no Gumbel draw can close, on either path.
logits = torch.zeros(num_reqs, vocab_size).cuda()
logits[:, 6] = 100.0
logits[:, 0] = float(step) + 1.0
step_logits.append(logits)
sampled = draft_watermarker.sample(
logits,
idx_mapping,
temperature=torch.ones(num_reqs).cuda(),
seed=torch.zeros(num_reqs, dtype=torch.int64).cuda(),
pos=torch.full((num_reqs,), 5 + step, dtype=torch.int64).cuda(),
apply_temperature=True,
is_drafting=True,
logits_cache_col=torch.tensor(step).cuda(),
logits_cache=draft_logits,
use_fp64=False,
)
assert torch.equal(sampled.cpu(), torch.tensor([6, 6]))
# Step 0: request 0's [5, 6] is committed at position 2, request 1's [6, 3]
# is new. Step 1: [6, 6] and [3, 6] are both new. Step 2: request 0's
# [6, 6] repeats its own step-1 context and request 1's repeats position 4.
assert [mask.tolist() for mask in skip_masks] == [
[True, False],
[False, False],
[True, True],
]
assert torch.equal(
draft_watermarker.prior_contexts[:num_reqs].cpu(),
torch.tensor([[[5, 6], [6, 6], [6, 6]], [[6, 3], [3, 6], [6, 6]]]),
)
assert torch.equal(
draft_watermarker.contexts[:num_reqs].cpu(), torch.tensor([[6, 6], [6, 6]])
)
for step, logits in enumerate(step_logits):
# The 0-d draft step selects the logits cache column.
assert torch.equal(draft_logits[:, step, :vocab_size], logits)
assert not draft_logits[:, :, -1].any()
def test_dspark_reduced_vocab_draft_sampler_applies_watermarking(monkeypatch):
speculator = object.__new__(DSparkSpeculator)
speculator.draft_logits = torch.zeros(2, 1, 8)
@@ -982,15 +1511,23 @@ def test_dspark_reduced_vocab_draft_sampler_applies_watermarking(monkeypatch):
speculator.acceptance_estimator = None
watermark_logits: list[torch.Tensor] = []
def sample(logits, sampled, idx_map, temperature):
def sample(
logits,
idx_mapping,
temperature,
seed,
pos,
apply_temperature,
is_drafting,
logits_cache=None,
logits_cache_col=None,
use_fp64=False,
logits_cache_source=None,
):
watermark_logits.append(logits.clone())
return sampled + 1
return torch.tensor([4, 5])
speculator.draft_watermarker = SimpleNamespace(sample=sample)
monkeypatch.setattr(
"vllm.v1.worker.gpu.spec_decode.dspark.speculator.gumbel_sample",
lambda *args, **kwargs: torch.tensor([3, 4]),
)
sampled = speculator._sample_logits(
torch.tensor([[10.0, 20.0], [30.0, 40.0]]),
-7
View File
@@ -1344,13 +1344,6 @@ class VllmConfig:
"share of output tokens supplied by accepted drafts.",
scope="global",
)
if watermark_config.deduplicate_contexts != "none":
logger.warning_once(
"Context deduplication is not supported with speculative "
"decoding and will not be applied to accepted drafts, "
"rejection-recovery tokens, or bonus tokens.",
scope="global",
)
if watermark_config.algorithm == "dual_key_gumbel" and (
watermark_config.alpha != get_field(WatermarkConfig, "alpha").default
):
+11 -2
View File
@@ -119,8 +119,9 @@ class GPUWatermarkSampler(Sampler):
self,
expanded_idx_mapping: torch.Tensor,
contexts: torch.Tensor,
expanded_local_pos: torch.Tensor | None = None,
) -> torch.Tensor:
return repeated_context_mask(
repeated = repeated_context_mask(
self.req_states.all_token_ids.gpu,
expanded_idx_mapping,
self.req_states.prompt_len.gpu,
@@ -128,8 +129,16 @@ class GPUWatermarkSampler(Sampler):
contexts,
self.deduplicate_contexts_max_history,
include_prompt=self.deduplicate_contexts == "all",
skip_partial_context=self.deduplicate_contexts == "all",
skip_partial_context=(
self.deduplicate_contexts == "all" and expanded_local_pos is None
),
history_offsets=expanded_local_pos,
local_positions=expanded_local_pos,
num_speculative_steps=(
self.num_speculative_tokens if expanded_local_pos is not None else 0
),
)
return repeated
def _get_contexts(
self,
+4
View File
@@ -94,6 +94,10 @@ class GumbelWatermarker(Watermarker):
seeds=random_sampler.seeds,
positions=random_sampler.positions,
use_fp64=random_sampler.use_fp64,
is_drafting=random_sampler.is_drafting,
logits_cache=random_sampler.logits_cache,
logits_cache_col=random_sampler.logits_cache_col,
logits_cache_source=random_sampler.logits_cache_source,
)
return WatermarkSample(token_ids, logits)
+112 -10
View File
@@ -3,14 +3,16 @@
import torch
from vllm.config.watermarking import WatermarkConfig
from vllm.config.watermarking import WatermarkConfig, WatermarkContextScope
from vllm.v1.watermarking.factory import create_watermarker
from vllm.v1.watermarking.gumbel import GumbelWatermarker
from vllm.v1.watermarking.prfs import PhiloxPRF
from vllm.v1.watermarking.watermarker import (
RandomSampler,
SupportsSpeculativeDecoding,
Watermarker,
)
from vllm.v1.worker.gpu.sample.watermark import draft_watermarking_mask
from vllm.v1.worker.gpu.spec_decode.rejection_sampler_utils import rejection_sample
@@ -20,8 +22,14 @@ class DraftWatermarker:
watermarker: Watermarker,
max_num_reqs: int,
device: torch.device,
num_speculative_steps: int,
deduplicate_contexts: WatermarkContextScope,
deduplicate_contexts_max_history: int | None,
) -> None:
self.watermarker = watermarker
self.num_speculative_steps = num_speculative_steps
self.deduplicate_contexts = deduplicate_contexts
self.deduplicate_contexts_max_history = deduplicate_contexts_max_history
self.contexts = torch.zeros(
max_num_reqs,
watermarker.context_width,
@@ -29,30 +37,112 @@ class DraftWatermarker:
device=device,
)
self.enabled = torch.zeros(max_num_reqs, dtype=torch.bool, device=device)
self.prior_contexts = torch.zeros(
max_num_reqs,
num_speculative_steps,
watermarker.context_width,
dtype=torch.int64,
device=device,
)
self.all_token_ids: torch.Tensor | None = None
self.prompt_lens: torch.Tensor | None = None
self.total_lens: torch.Tensor | None = None
def prepare(self, contexts: torch.Tensor, enabled: torch.Tensor) -> None:
def prepare(
self,
contexts: torch.Tensor,
enabled: torch.Tensor,
all_token_ids: torch.Tensor,
prompt_lens: torch.Tensor,
total_lens: torch.Tensor,
) -> None:
num_reqs = contexts.shape[0]
self.contexts[:num_reqs].copy_(contexts)
self.enabled[:num_reqs].copy_(enabled)
self.all_token_ids = all_token_ids
self.prompt_lens = prompt_lens
self.total_lens = total_lens
def _sampling_state(
self,
logits: torch.Tensor,
idx_mapping: torch.Tensor,
request_temperatures: torch.Tensor,
draft_step: int | torch.Tensor,
contexts: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
num_rows = logits.shape[0]
steps = torch.as_tensor(draft_step, device=logits.device, dtype=torch.int64)
if steps.ndim == 0:
steps = steps.expand(num_rows)
enabled = self.enabled[:num_rows] & (request_temperatures != 0)
if self.deduplicate_contexts == "none":
return steps, enabled
assert self.all_token_ids is not None
assert self.prompt_lens is not None
assert self.total_lens is not None
enabled = draft_watermarking_mask(
self.all_token_ids,
idx_mapping,
self.prompt_lens,
self.total_lens,
self.prior_contexts,
contexts,
steps,
enabled,
self.deduplicate_contexts_max_history,
include_prompt=self.deduplicate_contexts == "all",
)
return steps, enabled
def sample(
self,
logits: torch.Tensor,
ordinary_sampled: torch.Tensor,
idx_mapping: torch.Tensor,
temperature: torch.Tensor,
seed: torch.Tensor,
pos: torch.Tensor,
apply_temperature: bool,
is_drafting: bool,
logits_cache: torch.Tensor | None = None,
logits_cache_col: torch.Tensor | None = None,
use_fp64: bool = False,
logits_cache_source: torch.Tensor | None = None,
) -> torch.Tensor:
assert apply_temperature
assert is_drafting
assert logits_cache is not None
assert logits_cache_col is not None
num_rows = logits.shape[0]
request_temperatures = temperature[idx_mapping]
contexts = self.contexts[:num_rows]
steps, enabled = self._sampling_state(
logits, idx_mapping, request_temperatures, logits_cache_col, contexts
)
processed_logits = logits / torch.where(
request_temperatures == 0, 1, request_temperatures
).unsqueeze(-1)
watermarked = self.watermarker.sample(
random_sampler = RandomSampler(
expanded_idx_mapping=idx_mapping,
temperatures=temperature,
seeds=seed,
positions=pos,
use_fp64=use_fp64,
is_drafting=True,
logits_cache=logits_cache,
logits_cache_col=steps,
logits_cache_source=(
logits if logits_cache_source is None else logits_cache_source
),
)
sampled = self.watermarker.sample(
processed_logits,
self.contexts[: logits.shape[0]],
contexts,
random_sampler=random_sampler,
skip_mask=~enabled,
).token_ids
enabled = self.enabled[: logits.shape[0]] & (request_temperatures != 0)
sampled = torch.where(enabled, watermarked, ordinary_sampled)
contexts = self.contexts[: logits.shape[0]]
contexts.copy_(torch.cat((contexts[:, 1:], sampled.unsqueeze(-1)), dim=-1))
return sampled
@@ -68,9 +158,19 @@ def create_speculative_draft_watermarker(
max_num_reqs: int,
device: torch.device,
allow_target_only: bool,
num_speculative_steps: int = 1,
deduplicate_contexts: WatermarkContextScope = "none",
deduplicate_contexts_max_history: int | None = 8192,
) -> DraftWatermarker | None:
if isinstance(watermarker, SupportsSpeculativeDecoding):
return DraftWatermarker(watermarker.draft_watermarker, max_num_reqs, device)
return DraftWatermarker(
watermarker.draft_watermarker,
max_num_reqs,
device,
num_speculative_steps,
deduplicate_contexts,
deduplicate_contexts_max_history,
)
if allow_target_only:
return None
raise ValueError(
@@ -130,9 +230,10 @@ def watermarked_rejection_sample(
contexts: torch.Tensor,
watermarking: torch.Tensor,
watermarker: Watermarker,
watermarking_skip_mask: torch.Tensor | None = None,
use_fp64: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Use the target key for rejection recovery and bonus tokens."""
"""Use the target key except at contexts selected for ordinary sampling."""
assert contexts.shape == (target_logits.shape[0], watermarker.context_width)
return rejection_sample(
target_logits,
@@ -149,5 +250,6 @@ def watermarked_rejection_sample(
use_fp64=use_fp64,
contexts=contexts,
watermarking=watermarking,
watermarking_skip_mask=watermarking_skip_mask,
watermark_key=_resolve_watermark_key(watermarker),
)
+8 -1
View File
@@ -23,6 +23,10 @@ class RandomSampler:
seeds: torch.Tensor
positions: torch.Tensor
use_fp64: bool = False
is_drafting: bool = False
logits_cache: torch.Tensor | None = None
logits_cache_col: torch.Tensor | None = None
logits_cache_source: torch.Tensor | None = None
def __call__(self, logits: torch.Tensor) -> torch.Tensor:
return gumbel_sample(
@@ -32,7 +36,10 @@ class RandomSampler:
self.seeds,
self.positions,
apply_temperature=False,
is_drafting=False,
is_drafting=self.is_drafting,
logits_cache=self.logits_cache,
logits_cache_col=self.logits_cache_col,
logits_cache_source=self.logits_cache_source,
use_fp64=self.use_fp64,
)
+2 -4
View File
@@ -914,8 +914,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
spec_hidden_states = pre_hc_hidden_states[: hidden_states.shape[0]] # type: ignore[union-attr]
if isinstance(self.sampler, GPUWatermarkSampler):
self.speculator.prepare_watermarking(
self.sampler._get_contexts(input_batch.idx_mapping),
self.sampler.watermarking.gpu[input_batch.idx_mapping],
self.sampler, input_batch.idx_mapping
)
with use_workspace_lane(self._draft_workspace_lane):
self.speculator.propose(
@@ -2194,8 +2193,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
spec_hidden_states = pre_hc_hidden_states[: draft_hidden_states.size(0)]
if isinstance(self.sampler, GPUWatermarkSampler):
self.speculator.prepare_watermarking(
self.sampler._get_contexts(input_batch.idx_mapping),
self.sampler.watermarking.gpu[input_batch.idx_mapping],
self.sampler, input_batch.idx_mapping
)
with use_workspace_lane(self._draft_workspace_lane):
draft_tokens = self.speculator.propose(
+29 -1
View File
@@ -220,6 +220,8 @@ def gumbel_block_argmax(
logits_cache_stride_0,
logits_cache_stride_1,
logits_cache_col_ptr,
logits_cache_source_ptr,
logits_cache_source_stride,
vocab_size,
IS_DRAFTING: tl.constexpr,
APPLY_TEMPERATURE: tl.constexpr,
@@ -240,12 +242,16 @@ def gumbel_block_argmax(
col = tl.load(logits_cache_col_ptr + token_idx)
else:
col = tl.load(logits_cache_col_ptr)
cached_logits = tl.load(
logits_cache_source_ptr + token_idx * logits_cache_source_stride + block,
mask=mask,
)
tl.store(
logits_cache_ptr
+ req_state_idx * logits_cache_stride_0
+ col * logits_cache_stride_1
+ block,
logits,
cached_logits,
mask=mask & is_valid_req,
)
@@ -275,6 +281,8 @@ def _gumbel_sample_kernel(
logits_cache_stride_0,
logits_cache_stride_1,
logits_cache_col_ptr,
logits_cache_source_ptr,
logits_cache_source_stride,
logits_ptr,
logits_stride,
expanded_idx_mapping_ptr,
@@ -312,6 +320,8 @@ def _gumbel_sample_kernel(
logits_cache_stride_0,
logits_cache_stride_1,
logits_cache_col_ptr,
logits_cache_source_ptr,
logits_cache_source_stride,
vocab_size,
IS_DRAFTING=IS_DRAFTING,
APPLY_TEMPERATURE=APPLY_TEMPERATURE,
@@ -334,6 +344,7 @@ def gumbel_sample(
logits_cache: torch.Tensor | None = None, # [max_num_reqs, num_cols, vocab_size]
logits_cache_col: torch.Tensor | None = None, # scalar or [num_tokens]
use_fp64: bool = False,
logits_cache_source: torch.Tensor | None = None,
) -> torch.Tensor:
# Enforce contiguity on non-strided input tensors
expanded_idx_mapping = expanded_idx_mapping.contiguous()
@@ -342,11 +353,26 @@ def gumbel_sample(
logits_cache_col = logits_cache_col.contiguous()
num_tokens, vocab_size = logits.shape
if logits_cache is not None:
if logits_cache_source is None:
logits_cache_source = logits
assert logits_cache_source.shape == logits.shape, (
"logits cache source must match sampled logits shape"
)
assert logits_cache_source.device == logits.device, (
"logits cache source must be on the sampled logits device"
)
assert logits_cache_source.dtype == logits_cache.dtype, (
"logits cache source and destination must have the same dtype"
)
assert logits_cache.size(-1) >= vocab_size, (
f"draft logits cache vocab dim ({logits_cache.size(-1)}) is narrower "
f"than the sampled logits ({vocab_size}). Cached logits would be "
"truncated."
)
elif logits_cache_source is not None:
raise ValueError("logits_cache_source requires logits_cache")
if logits_cache_source is not None and logits_cache_source.stride(-1) != 1:
logits_cache_source = logits_cache_source.contiguous()
BLOCK_SIZE = 1024
num_blocks = triton.cdiv(vocab_size, BLOCK_SIZE)
local_argmax = logits.new_empty(num_tokens, num_blocks, dtype=torch.int64)
@@ -362,6 +388,8 @@ def gumbel_sample(
logits_cache.stride(0) if logits_cache is not None else 0,
logits_cache.stride(1) if logits_cache is not None else 0,
logits_cache_col,
logits_cache_source,
logits_cache_source.stride(0) if logits_cache_source is not None else 0,
logits,
logits.stride(0),
expanded_idx_mapping,
+315 -7
View File
@@ -51,12 +51,25 @@ def _repeated_context_mask_kernel(
req_indices_ptr,
prompt_lens_ptr,
total_lens_ptr,
history_offsets_ptr,
history_offsets_stride,
contexts_ptr,
context_stride,
local_positions_ptr,
prior_contexts_ptr,
prior_contexts_stride_0,
prior_contexts_stride_1,
steps_ptr,
steps_stride,
enabled_ptr,
CONTEXT_WIDTH: tl.constexpr,
NUM_SPECULATIVE_STEPS: tl.constexpr,
MAX_HISTORY: tl.constexpr,
INCLUDE_PROMPT: tl.constexpr,
SKIP_PARTIAL_CONTEXT: tl.constexpr,
HAS_HISTORY_OFFSETS: tl.constexpr,
SPECULATIVE_CONTEXTS: tl.constexpr,
DRAFT_CONTEXTS: tl.constexpr,
BLOCK: tl.constexpr,
):
row = tl.program_id(0).to(tl.int64)
@@ -71,12 +84,16 @@ def _repeated_context_mask_kernel(
return
sequence_start = 0 if INCLUDE_PROMPT else prompt_len
history_len = total_len - sequence_start
history_offset = 0
if HAS_HISTORY_OFFSETS:
history_offset = tl.load(history_offsets_ptr + row * history_offsets_stride)
offsets = tl.arange(0, BLOCK)
repeated = tl.full((), 0, tl.int32)
scan_start = 0
if MAX_HISTORY > 0:
scan_start = tl.maximum(history_len - MAX_HISTORY, 0)
remaining_history = tl.maximum(MAX_HISTORY - history_offset, 0)
scan_start = tl.maximum(history_len - remaining_history, 0)
# Aligned loop bounds let the per-lane offsets fold into immediates.
aligned_start = (scan_start // BLOCK) * BLOCK
for block_start in tl.range(aligned_start, history_len, BLOCK):
@@ -96,6 +113,93 @@ def _repeated_context_mask_kernel(
)
matches &= historical_token == context_token
repeated |= tl.max(matches.to(tl.int32), axis=0)
if SPECULATIVE_CONTEXTS:
local_pos = tl.load(local_positions_ptr + row)
if INCLUDE_PROMPT:
partial_context = tl.full((), 0, tl.int32)
for context_offset in range(CONTEXT_WIDTH):
context_token = tl.load(
contexts_ptr + row * context_stride + context_offset
)
partial_context |= context_token < 0
repeated |= partial_context
for offset in range(1, NUM_SPECULATIVE_STEPS + 1):
prior_row = row - offset
in_history = (prior_row >= 0) & (local_pos >= offset)
if MAX_HISTORY > 0:
in_history &= offset <= MAX_HISTORY
prior_req_idx = tl.load(
req_indices_ptr + tl.maximum(prior_row, 0),
mask=in_history,
other=-1,
)
prior_local_pos = tl.load(
local_positions_ptr + tl.maximum(prior_row, 0),
mask=in_history,
other=-1,
)
matches = (
in_history
& (req_idx == prior_req_idx)
& (local_pos == prior_local_pos + offset)
)
for context_offset in range(CONTEXT_WIDTH):
context_token = tl.load(
contexts_ptr + row * context_stride + context_offset
)
prior_token = tl.load(
contexts_ptr
+ tl.maximum(prior_row, 0) * context_stride
+ context_offset,
mask=in_history,
other=-2,
)
matches &= context_token == prior_token
repeated |= matches
if DRAFT_CONTEXTS:
step = tl.load(steps_ptr + row * steps_stride).to(tl.int64)
partial_context = tl.full((), 0, tl.int32)
for context_offset in range(CONTEXT_WIDTH):
context_token = tl.load(
contexts_ptr + row * context_stride + context_offset
)
partial_context |= context_token < 0
if INCLUDE_PROMPT:
repeated |= partial_context
for prior_step in range(NUM_SPECULATIVE_STEPS):
in_history = prior_step < step
if MAX_HISTORY > 0:
in_history &= prior_step >= step - MAX_HISTORY
matches = in_history
for context_offset in range(CONTEXT_WIDTH):
context_token = tl.load(
contexts_ptr + row * context_stride + context_offset
)
prior_token = tl.load(
prior_contexts_ptr
+ row * prior_contexts_stride_0
+ prior_step * prior_contexts_stride_1
+ context_offset
)
matches &= context_token == prior_token
repeated |= matches
for context_offset in range(CONTEXT_WIDTH):
context_token = tl.load(
contexts_ptr + row * context_stride + context_offset
)
tl.store(
prior_contexts_ptr
+ row * prior_contexts_stride_0
+ step * prior_contexts_stride_1
+ context_offset,
context_token,
)
enabled = tl.load(enabled_ptr + row)
repeated = enabled & (repeated == 0)
tl.store(output_ptr + row, repeated)
@@ -108,6 +212,7 @@ def _repeated_context_mask_cpu(
max_history: int | None = None,
include_prompt: bool = False,
skip_partial_context: bool = False,
history_offsets: torch.Tensor | None = None,
) -> torch.Tensor:
"""Reference implementation; the parity tests check the Triton kernel against it."""
repeated = torch.zeros(len(req_indices), dtype=torch.bool)
@@ -124,8 +229,11 @@ def _repeated_context_mask_cpu(
history_tokens = all_token_ids[req_idx, sequence_start:total_len].tolist()
prefix = [-1] * contexts.shape[-1]
current_context = tuple(contexts[row].tolist())
history_offset = 0 if history_offsets is None else int(history_offsets[row])
history_start = (
max(0, len(history_tokens) - max_history) if max_history is not None else 0
max(0, len(history_tokens) - max(0, max_history - history_offset))
if max_history is not None
else 0
)
for history_pos, token_id in enumerate(history_tokens):
if history_pos >= history_start and (
@@ -146,6 +254,9 @@ def repeated_context_mask(
max_history: int | None = None,
include_prompt: bool = False,
skip_partial_context: bool = False,
history_offsets: torch.Tensor | None = None,
local_positions: torch.Tensor | None = None,
num_speculative_steps: int = 0,
) -> torch.Tensor:
"""Return, per row, whether the row's context already occurred in its history.
@@ -163,12 +274,23 @@ def repeated_context_mask(
include_prompt: Search the prompt as well as the generated tokens.
skip_partial_context: Mark contexts containing start padding so they use
ordinary sampling.
history_offsets: Number of newer, non-committed positions preceding each
context. These positions count toward `max_history`. Must be 1-D
over rows; a stride-0 broadcast row is fine.
local_positions: Position within each request's speculative block. When
provided, compare each context with earlier contexts in that block.
num_speculative_steps: Maximum number of draft tokens in the block.
"""
if max_history is not None and max_history < 1:
raise ValueError("max_history must be positive or None")
if (local_positions is None) != (num_speculative_steps == 0):
raise ValueError(
"local_positions and positive num_speculative_steps must be "
"provided together"
)
if all_token_ids.device.type == "cpu":
return _repeated_context_mask_cpu(
repeated = _repeated_context_mask_cpu(
all_token_ids,
req_indices,
prompt_lens,
@@ -177,10 +299,42 @@ def repeated_context_mask(
max_history,
include_prompt,
skip_partial_context,
history_offsets,
)
if local_positions is not None:
max_offset = num_speculative_steps + 1
if max_history is not None:
max_offset = min(max_offset, max_history + 1)
for offset in range(1, max_offset):
prior_contexts = torch.cat(
(contexts[:offset], contexts[:-offset]), dim=0
)
prior_requests = torch.cat(
(req_indices[:offset], req_indices[:-offset]), dim=0
)
prior_local_pos = torch.cat(
(local_positions[:offset], local_positions[:-offset]), dim=0
)
repeated |= (
(req_indices >= 0)
& (local_positions >= offset)
& (req_indices == prior_requests)
& (local_positions == prior_local_pos + offset)
& (contexts == prior_contexts).all(dim=-1)
)
if include_prompt:
repeated |= (contexts < 0).any(dim=-1)
return repeated
if contexts.stride(-1) != 1:
contexts = contexts.contiguous()
# Only history_offsets passes its row stride to the kernel.
req_indices = req_indices.contiguous()
if local_positions is not None:
local_positions = local_positions.contiguous()
# `prompt_lens` and `total_lens` are indexed flat by request slot, and
# `all_token_ids` takes a row stride but is read flat within the row. Those
# three are contiguous by contract; every caller owns them outright.
repeated = torch.empty(len(req_indices), dtype=torch.bool, device=contexts.device)
_repeated_context_mask_kernel[(len(req_indices),)](
repeated,
@@ -189,17 +343,104 @@ def repeated_context_mask(
req_indices,
prompt_lens,
total_lens,
history_offsets,
0 if history_offsets is None else history_offsets.stride(0),
contexts,
contexts.stride(0),
local_positions,
None,
0,
0,
None,
0,
None,
CONTEXT_WIDTH=contexts.shape[-1],
NUM_SPECULATIVE_STEPS=num_speculative_steps,
MAX_HISTORY=0 if max_history is None else max_history,
INCLUDE_PROMPT=include_prompt,
SKIP_PARTIAL_CONTEXT=skip_partial_context,
HAS_HISTORY_OFFSETS=history_offsets is not None,
SPECULATIVE_CONTEXTS=local_positions is not None,
DRAFT_CONTEXTS=False,
BLOCK=512,
)
return repeated
def draft_watermarking_mask(
all_token_ids: torch.Tensor,
req_indices: torch.Tensor,
prompt_lens: torch.Tensor,
total_lens: torch.Tensor,
prior_contexts: torch.Tensor,
contexts: torch.Tensor,
steps: torch.Tensor,
enabled: torch.Tensor,
max_history: int | None,
include_prompt: bool,
) -> torch.Tensor:
if contexts.device.type == "cpu":
repeated = repeated_context_mask(
all_token_ids,
req_indices,
prompt_lens,
total_lens,
contexts,
max_history,
include_prompt=include_prompt,
history_offsets=steps,
)
prior_steps = torch.arange(prior_contexts.shape[1]).unsqueeze(0)
in_history = prior_steps < steps.unsqueeze(1)
if max_history is not None:
in_history &= prior_steps >= steps.unsqueeze(1) - max_history
repeated |= (
(prior_contexts[: len(contexts)] == contexts.unsqueeze(1)).all(dim=-1)
& in_history
).any(dim=-1)
if include_prompt:
repeated |= (contexts < 0).any(dim=-1)
row_indices = torch.arange(len(contexts))
prior_contexts.index_put_((row_indices, steps), contexts)
return enabled & ~repeated
if contexts.stride(-1) != 1:
contexts = contexts.contiguous()
# Only `steps` passes its row stride to the kernel.
req_indices = req_indices.contiguous()
enabled = enabled.contiguous()
active = torch.empty(len(contexts), dtype=torch.bool, device=contexts.device)
_repeated_context_mask_kernel[(len(contexts),)](
active,
all_token_ids,
all_token_ids.stride(0),
req_indices,
prompt_lens,
total_lens,
steps,
steps.stride(0),
contexts,
contexts.stride(0),
None,
prior_contexts,
prior_contexts.stride(0),
prior_contexts.stride(1),
steps,
steps.stride(0),
enabled,
NUM_SPECULATIVE_STEPS=prior_contexts.shape[1],
CONTEXT_WIDTH=contexts.shape[1],
MAX_HISTORY=0 if max_history is None else max_history,
INCLUDE_PROMPT=include_prompt,
SKIP_PARTIAL_CONTEXT=False,
HAS_HISTORY_OFFSETS=True,
SPECULATIVE_CONTEXTS=False,
DRAFT_CONTEXTS=True,
BLOCK=512,
)
return active
@triton.jit
def _mulhilo32(multiplier: tl.constexpr, value):
high = tl_math.umulhi(multiplier, value)
@@ -351,28 +592,58 @@ def _philox_gumbel_kernel(
seeds_ptr,
pos_ptr,
temp_ptr,
logits_cache_ptr,
logits_cache_stride_0,
logits_cache_stride_1,
logits_cache_col_ptr,
logits_cache_source_ptr,
logits_cache_source_stride,
key_0_value,
key_1_value,
vocab_size,
CONTEXT_WIDTH: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
USE_FP64: tl.constexpr,
IS_DRAFTING: tl.constexpr,
PER_TOKEN_COL: tl.constexpr,
):
row = tl.program_id(0).to(tl.int64)
block_index = tl.program_id(1)
groups = block_index * (BLOCK_SIZE // 4) + tl.arange(0, BLOCK_SIZE // 4)
candidate = block_index * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = candidate < vocab_size
if skip_mask_ptr is not None:
req_state_idx = row
valid_req = True
if expanded_idx_mapping_ptr is not None:
req_state_idx = tl.load(expanded_idx_mapping_ptr + row).to(tl.int64)
valid_req = req_state_idx >= 0
if logits_cache_ptr is not None:
if PER_TOKEN_COL:
col = tl.load(logits_cache_col_ptr + row)
else:
col = tl.load(logits_cache_col_ptr)
cached_logits = tl.load(
logits_cache_source_ptr + row * logits_cache_source_stride + candidate,
mask=mask,
)
tl.store(
logits_cache_ptr
+ req_state_idx * logits_cache_stride_0
+ col * logits_cache_stride_1
+ candidate,
cached_logits,
mask=mask & valid_req,
)
if skip_mask_ptr is not None: # noqa: SIM102
if tl.load(skip_mask_ptr + row):
temp = tl.load(temp_ptr + req_state_idx, mask=valid_req, other=0.0).to(
tl.float32
)
seed = tl.load(seeds_ptr + req_state_idx, mask=valid_req, other=0)
pos = tl.load(pos_ptr + row)
candidate = block_index * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = candidate < vocab_size
logits_row = logits_ptr + row * logits_stride
logits = tl.load(logits_row + candidate, mask=mask, other=float("-inf")).to(
tl.float32
@@ -384,7 +655,7 @@ def _philox_gumbel_kernel(
seed,
pos,
temp,
IS_DRAFTING=False,
IS_DRAFTING=IS_DRAFTING,
USE_FP64=USE_FP64,
APPLY_TEMPERATURE=False,
)
@@ -443,12 +714,37 @@ def philox_gumbel_sample(
seeds: torch.Tensor | None = None,
positions: torch.Tensor | None = None,
use_fp64: bool = False,
is_drafting: bool = False,
logits_cache: torch.Tensor | None = None,
logits_cache_col: torch.Tensor | None = None,
logits_cache_source: torch.Tensor | None = None,
) -> torch.Tensor:
sampling_state = (expanded_idx_mapping, temperatures, seeds, positions)
if skip_mask is None and any(value is not None for value in sampling_state):
raise ValueError("sampling state requires skip_mask")
if skip_mask is not None and any(value is None for value in sampling_state):
raise ValueError("skip_mask requires complete sampling state")
if logits_cache is not None:
if logits_cache_col is None or logits_cache_source is None:
raise ValueError("logits cache requires its column and source logits")
if expanded_idx_mapping is None:
raise ValueError("logits cache requires expanded_idx_mapping")
assert logits_cache_source.shape == logits.shape, (
"logits cache source must match sampled logits shape"
)
assert logits_cache_source.device == logits.device, (
"logits cache source must be on the sampled logits device"
)
assert logits_cache_source.dtype == logits_cache.dtype, (
"logits cache source and destination must have the same dtype"
)
assert logits_cache.size(-1) >= logits.shape[-1], (
f"draft logits cache vocab dim ({logits_cache.size(-1)}) is narrower "
f"than the sampled logits ({logits.shape[-1]}). Cached logits would be "
"truncated."
)
elif logits_cache_col is not None or logits_cache_source is not None:
raise ValueError("logits cache column and source require logits_cache")
if logits.stride(-1) != 1:
logits = logits.contiguous()
@@ -460,6 +756,10 @@ def philox_gumbel_sample(
expanded_idx_mapping = expanded_idx_mapping.contiguous()
if positions is not None:
positions = positions.contiguous()
if logits_cache_col is not None:
logits_cache_col = logits_cache_col.contiguous()
if logits_cache_source is not None and logits_cache_source.stride(-1) != 1:
logits_cache_source = logits_cache_source.contiguous()
num_tokens, vocab_size = logits.shape
block_size = 1024
num_blocks = triton.cdiv(vocab_size, block_size)
@@ -483,12 +783,20 @@ def philox_gumbel_sample(
seeds,
positions,
temperatures,
logits_cache,
logits_cache.stride(0) if logits_cache is not None else 0,
logits_cache.stride(1) if logits_cache is not None else 0,
logits_cache_col,
logits_cache_source,
logits_cache_source.stride(0) if logits_cache_source is not None else 0,
key & _UINT32_MASK_VALUE,
key >> 32,
vocab_size,
CONTEXT_WIDTH=contexts.shape[-1],
BLOCK_SIZE=block_size,
USE_FP64=use_fp64,
IS_DRAFTING=is_drafting,
PER_TOKEN_COL=logits_cache_col is not None and logits_cache_col.dim() > 0,
)
max_block_index = local_max.argmax(dim=-1, keepdim=True)
return local_argmax.gather(dim=-1, index=max_block_index).view(-1)
@@ -135,7 +135,12 @@ class DSparkSpeculator(DFlashSpeculator):
# sample_pos is the predicted token's position P. Sampling keys a draw
# by the position before the sampled token, P-1.
sampled = gumbel_sample(
sampler = (
gumbel_sample
if self.draft_watermarker is None
else self.draft_watermarker.sample
)
return sampler(
logits,
idx_map,
self.temperature,
@@ -147,11 +152,6 @@ class DSparkSpeculator(DFlashSpeculator):
logits_cache_col=self._step_cols[step],
use_fp64=self.use_fp64_gumbel,
)
if self.draft_watermarker is not None:
sampled = self.draft_watermarker.sample(
logits, sampled, idx_map, self.temperature
)
return sampled
def _sample_sequential(self, num_reqs: int, head_hidden: torch.Tensor) -> None:
if self._draft_topk is not None:
@@ -226,11 +226,18 @@ class RejectionSampler:
if self.watermark_key is None:
return {}
assert isinstance(self.sampler, GPUWatermarkSampler)
contexts = self.sampler._get_contexts(
expanded_idx_mapping, expanded_local_pos, draft_sampled
)
watermarking_skip_mask = None
if self.sampler.deduplicate_contexts != "none":
watermarking_skip_mask = self.sampler._get_repeated_contexts(
expanded_idx_mapping, contexts, expanded_local_pos
)
return {
"contexts": self.sampler._get_contexts(
expanded_idx_mapping, expanded_local_pos, draft_sampled
),
"contexts": contexts,
"watermarking": self.sampler.watermarking.gpu,
"watermarking_skip_mask": watermarking_skip_mask,
"watermark_key": self.watermark_key,
}
@@ -564,7 +564,6 @@ def _rejection_kernel(
# stores the target argmax upon first rejection, so it rejects the
# placeholder via `accepted` instead.
verifying &= is_valid_draft
if verifying:
pos = tl.load(pos_ptr + logit_idx)
u = tl_rand32(seed, pos, includes_zero=False)
@@ -719,6 +718,8 @@ def _seeded_resample_argmax(
0, # logits_cache_stride_0
0, # logits_cache_stride_1
None, # logits_cache_col_ptr
None, # logits_cache_source_ptr
0, # logits_cache_source_stride
vocab_size,
IS_DRAFTING=False,
APPLY_TEMPERATURE=False,
@@ -766,6 +767,7 @@ def _resample_kernel(
contexts_stride,
# [max_num_reqs], uint8 view of a bool tensor
watermarking_ptr,
watermarking_skip_mask_ptr,
watermark_key_0,
watermark_key_1,
vocab_size,
@@ -775,6 +777,7 @@ def _resample_kernel(
USE_BLOCK_VERIFICATION: tl.constexpr,
CONTEXT_WIDTH: tl.constexpr,
WATERMARK: tl.constexpr,
DEDUPLICATE_CONTEXTS: tl.constexpr,
):
req_idx = tl.program_id(0)
resample_idx = tl.load(rejected_step_ptr + req_idx)
@@ -807,6 +810,25 @@ def _resample_kernel(
).to(tl.float32)
# Compute the residual logits to resample the rejected token from.
is_watermarked = False
if WATERMARK:
is_watermarked = (
tl.load(
watermarking_ptr + req_state_idx,
mask=req_state_idx >= 0,
other=0,
)
!= 0
)
skip_watermarking = False
if DEDUPLICATE_CONTEXTS:
skip_watermarking = (
(tl.load(watermarking_skip_mask_ptr + resample_token_idx) != 0)
& is_watermarked
& (temp != 0.0)
)
if is_bonus or not is_valid_rejected_draft:
# Bonus token (no rejections) or -1 placeholder token. In either case,
# directly use the target logits.
@@ -868,15 +890,7 @@ def _resample_kernel(
# Resample the rejected/bonus token.
if WATERMARK:
# Padded and greedy rows retain the stock draw.
is_watermarked = (
tl.load(
watermarking_ptr + req_state_idx,
mask=req_state_idx >= 0,
other=0,
)
!= 0
)
if is_watermarked & (temp != 0.0):
if is_watermarked & (temp != 0.0) & ~skip_watermarking:
watermark_value, idx = philox_gumbel_block_argmax(
residual_logits,
mask,
@@ -1023,6 +1037,7 @@ def rejection_sample(
use_block_verification: bool = False,
contexts: torch.Tensor | None = None,
watermarking: torch.Tensor | None = None,
watermarking_skip_mask: torch.Tensor | None = None,
watermark_key: int | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
assert target_logits.ndim == 2 and target_logits.stride(-1) == 1
@@ -1036,9 +1051,13 @@ def rejection_sample(
assert watermark == (watermarking is not None) == (watermark_key is not None), (
"contexts, watermarking and watermark_key must be set together."
)
assert watermarking_skip_mask is None or watermark, (
"watermarking_skip_mask requires watermarking."
)
contexts_stride = 0
context_width = 1
watermarking_bytes: torch.Tensor | None = None
watermarking_skip_mask_bytes: torch.Tensor | None = None
watermark_key_0 = 0
watermark_key_1 = 0
if contexts is not None:
@@ -1053,6 +1072,12 @@ def rejection_sample(
contexts_stride = contexts.stride(0)
context_width = contexts.shape[-1]
watermarking_bytes = watermarking.view(torch.uint8)
if watermarking_skip_mask is not None:
assert watermarking_skip_mask.shape == (num_logits,)
assert watermarking_skip_mask.dtype == torch.bool
if not watermarking_skip_mask.is_contiguous():
watermarking_skip_mask = watermarking_skip_mask.contiguous()
watermarking_skip_mask_bytes = watermarking_skip_mask.view(torch.uint8)
watermark_key_0 = watermark_key & 0xFFFFFFFF
watermark_key_1 = watermark_key >> 32
draft_logits_stride_0 = 0
@@ -1269,6 +1294,7 @@ def rejection_sample(
contexts,
contexts_stride,
watermarking_bytes,
watermarking_skip_mask_bytes,
watermark_key_0,
watermark_key_1,
vocab_size,
@@ -1278,6 +1304,7 @@ def rejection_sample(
USE_BLOCK_VERIFICATION=use_block_verification,
CONTEXT_WIDTH=context_width,
WATERMARK=watermark,
DEDUPLICATE_CONTEXTS=watermarking_skip_mask is not None,
)
# Insert the resampled tokens into the output sampled.
+20 -7
View File
@@ -38,6 +38,7 @@ from vllm.v1.worker.gpu.spec_decode.acceptance_estimator import (
from vllm.v1.worker.utils import AttentionGroup
if TYPE_CHECKING:
from vllm.v1.watermarking.gpu_sampler import GPUWatermarkSampler
from vllm.v1.worker.gpu.pcp_manager import PCPManager
logger = init_logger(__name__)
@@ -198,6 +199,9 @@ class DraftModelSpeculator(BaseSpeculator):
self.max_num_reqs,
device,
watermark_config.allow_target_only_watermarking,
self.num_speculative_steps,
watermark_config.deduplicate_contexts,
watermark_config.deduplicate_contexts_max_history,
)
self.supports_mm_inputs = False
@@ -411,7 +415,12 @@ class DraftModelSpeculator(BaseSpeculator):
) -> torch.Tensor:
if draft_logits is not None:
logits = self.model.compute_logits(hidden_states)
sampled = gumbel_sample(
sampler = (
gumbel_sample
if self.draft_watermarker is None
else self.draft_watermarker.sample
)
sampled = sampler(
logits,
idx_mapping,
temperature,
@@ -423,10 +432,6 @@ class DraftModelSpeculator(BaseSpeculator):
logits_cache_col=draft_step,
use_fp64=self.use_fp64_gumbel,
)
if self.draft_watermarker is not None:
sampled = self.draft_watermarker.sample(
logits, sampled, idx_mapping, temperature
)
elif self.use_local_argmax_reduction:
return self.model.get_top_tokens(hidden_states)
else:
@@ -462,11 +467,19 @@ class DraftModelSpeculator(BaseSpeculator):
self.acceptance_estimator.step(idx_mapping, num_sampled, num_rejected)
def prepare_watermarking(
self, contexts: torch.Tensor, watermarking: torch.Tensor
self,
sampler: "GPUWatermarkSampler",
idx_mapping: torch.Tensor,
) -> None:
if self.draft_watermarker is None:
return
self.draft_watermarker.prepare(contexts, watermarking)
self.draft_watermarker.prepare(
sampler._get_contexts(idx_mapping),
sampler.watermarking.gpu[idx_mapping],
sampler.req_states.all_token_ids.gpu,
sampler.req_states.prompt_len.gpu,
sampler.req_states.total_len.gpu,
)
def _copy_request_inputs(
self,