[watermarking] add context deduplication support to speculative decoding (#56807)

Signed-off-by: Raphael Rialland <raphael.rialland@mistral.ai>
Signed-off-by: Simon Veitner <sveitner@redhat.com>
Co-authored-by: Simon Veitner <sveitner@redhat.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Raphaël Rialland
2026-09-30 12:22:10 -04:00
committed by GitHub
co-authored by Simon Veitner Lucas Wilkinson
parent 73a5831127
commit 0f8b398158
17 changed files with 1379 additions and 88 deletions
+23 -2
View File
@@ -3526,7 +3526,6 @@ def test_target_only_gumbel_allows_speculative_decoding(caplog_vllm, disable_log
config._check_watermarking_unsupported()
assert "Target-only watermarking leaves accepted draft tokens" in caplog_vllm.text
assert "Context deduplication is not supported" in caplog_vllm.text
def test_speculative_watermarking_without_context_dedup_does_not_warn(
@@ -3543,10 +3542,32 @@ def test_speculative_watermarking_without_context_dedup_does_not_warn(
parallel_drafting=False,
)
caplog_vllm.clear()
with caplog_vllm.at_level(logging.WARNING):
config._check_watermarking_unsupported()
assert "Context deduplication is not supported" not in caplog_vllm.text
assert "dedup" not in caplog_vllm.text.lower()
def test_speculative_context_dedup_logs_no_unsupported_warning(
caplog_vllm, disable_log_dedup
):
config = _watermarked_vllm_config()
config.watermark_config = WatermarkConfig(
algorithm="dual_key_gumbel", key=42, deduplicate_contexts="single_turn"
)
config.speculative_config = SimpleNamespace(
method="mtp",
draft_sample_method="probabilistic",
rejection_sample_method="standard",
parallel_drafting=False,
)
caplog_vllm.clear()
with caplog_vllm.at_level(logging.WARNING):
config._check_watermarking_unsupported()
assert "dedup" not in caplog_vllm.text.lower()
def test_gumbel_rejects_speculative_decoding_without_target_only():
@@ -203,6 +203,182 @@ def test_watermarked_recovery_uses_target_key():
assert sampled[0, 0] == expected.token_ids[0]
def test_repeated_draft_context_does_not_change_acceptance():
torch.manual_seed(0)
num_trials = 8
# A draft distribution of its own, so the drafts are not accepted outright.
target_logits = torch.randn(64, device="cuda")
draft_logits = torch.randn(64, device="cuda")
inputs = _build_rejection_sample_inputs(
target_logits,
draft_logits,
num_speculative_steps=1,
temperature=1.0,
num_trials=num_trials,
)
expected, expected_num_sampled = rejection_sample(**inputs, num_speculative_steps=1)
sampled, num_sampled = watermarked_rejection_sample(
**inputs,
num_speculative_steps=1,
contexts=torch.tensor([[10, 11], [11, 12]], device="cuda").repeat(
num_trials, 1
),
watermarking=torch.ones(num_trials, dtype=torch.bool, device="cuda"),
watermarking_skip_mask=torch.tensor([True, False], device="cuda").repeat(
num_trials
),
watermarker=GumbelWatermarker(key=42, context_width=2),
)
# Some drafts must actually be rejected, or the ordinary-residual branch
# the skip mask selects is never reached.
assert (expected_num_sampled == 1).any()
torch.testing.assert_close(num_sampled, expected_num_sampled, rtol=0, atol=0)
torch.testing.assert_close(sampled[:, :1], expected[:, :1], rtol=0, atol=0)
def test_repeated_context_skip_mask_is_read_per_row_in_a_block():
"""A block of three drafts, accepted at row 0 and rejected at row 1.
The skip mask selects the sampler per row, so the row the target recovers
at decides which key draws the recovery token.
"""
num_speculative_steps = 3
target_logits = torch.tensor([float("-inf"), 0.0, 1.0, 2.0], device="cuda")
draft_logits = torch.tensor([30.0, -30.0, -30.0, -30.0], device="cuda")
inputs = _build_rejection_sample_inputs(
target_logits,
draft_logits,
num_speculative_steps=num_speculative_steps,
temperature=1.0,
num_trials=1,
)
# Row 0 drafts a token the target gives 0.67 and the draft e**-60, so it is
# accepted; row 1 drafts a token the target gives no mass, so it is not.
inputs["draft_sampled"][1] = 3
inputs["draft_sampled"][2] = 0
contexts = torch.tensor(
[[10, 11], [11, 3], [3, 0], [0, 1]], dtype=torch.int64, device="cuda"
)
expected, expected_num_sampled = rejection_sample(
**inputs, num_speculative_steps=num_speculative_steps
)
assert expected_num_sampled.item() == 2
assert expected[0, 0].item() == 3
watermarker = GumbelWatermarker(key=42, context_width=2)
skipped, skipped_num_sampled = watermarked_rejection_sample(
**inputs,
num_speculative_steps=num_speculative_steps,
contexts=contexts,
watermarking=torch.tensor([True], device="cuda"),
watermarking_skip_mask=torch.tensor([False, True, False, False], device="cuda"),
watermarker=watermarker,
)
assert skipped_num_sampled.item() == 2
torch.testing.assert_close(skipped[:, :2], expected[:, :2], rtol=0, atol=0)
watermarked, watermarked_num_sampled = watermarked_rejection_sample(
**inputs,
num_speculative_steps=num_speculative_steps,
contexts=contexts,
watermarking=torch.tensor([True], device="cuda"),
watermarking_skip_mask=torch.zeros(
num_speculative_steps + 1, dtype=torch.bool, device="cuda"
),
watermarker=watermarker,
)
# The residual equals the target to within e**-60, so the recovery token is
# the target key's draw on the rejected row's context.
recovery = GumbelWatermarker(key=42, context_width=2).sample(
target_logits.unsqueeze(0), contexts[1:2], lambda _: None
)
assert watermarked_num_sampled.item() == 2
assert watermarked[0, 0].item() == 3
assert watermarked[0, 1] == recovery.token_ids[0]
def test_repeated_recovery_context_uses_ordinary_residual_sample():
target_logits = torch.tensor([float("-inf"), 0.0, 1.0, 2.0], device="cuda")
draft_logits = torch.tensor([100.0, -100.0, -100.0, -100.0], device="cuda")
inputs = _build_rejection_sample_inputs(
target_logits,
draft_logits,
num_speculative_steps=1,
temperature=1.0,
num_trials=1,
)
inputs["draft_sampled"][1] = 0
expected, expected_num_sampled = rejection_sample(**inputs, num_speculative_steps=1)
sampled, num_sampled = watermarked_rejection_sample(
**inputs,
num_speculative_steps=1,
contexts=torch.tensor([[10, 11], [11, 0]], device="cuda"),
watermarking=torch.tensor([True], device="cuda"),
watermarking_skip_mask=torch.tensor([True, False], device="cuda"),
watermarker=GumbelWatermarker(key=42, context_width=2),
)
torch.testing.assert_close(num_sampled, expected_num_sampled, rtol=0, atol=0)
torch.testing.assert_close(sampled[:, :1], expected[:, :1], rtol=0, atol=0)
def test_repeated_bonus_context_uses_ordinary_target_sample():
target_logits = torch.randn(64, device="cuda")
inputs = _build_rejection_sample_inputs(
target_logits,
target_logits,
num_speculative_steps=1,
temperature=1.0,
num_trials=1,
)
expected, expected_num_sampled = rejection_sample(**inputs, num_speculative_steps=1)
sampled, num_sampled = watermarked_rejection_sample(
**inputs,
num_speculative_steps=1,
contexts=torch.tensor([[10, 11], [11, 12]], device="cuda"),
watermarking=torch.tensor([True], device="cuda"),
watermarking_skip_mask=torch.tensor([False, True], device="cuda"),
watermarker=GumbelWatermarker(key=42, context_width=2),
)
torch.testing.assert_close(num_sampled, expected_num_sampled, rtol=0, atol=0)
torch.testing.assert_close(sampled[:, :2], expected[:, :2], rtol=0, atol=0)
def test_repeated_context_does_not_change_unwatermarked_or_greedy_rows():
target_logits = torch.randn(64, device="cuda")
inputs = _build_rejection_sample_inputs(
target_logits,
target_logits,
num_speculative_steps=1,
temperature=1.0,
num_trials=2,
)
inputs["temperature"][1] = 0
expected, expected_num_sampled = rejection_sample(**inputs, num_speculative_steps=1)
sampled, num_sampled = watermarked_rejection_sample(
**inputs,
num_speculative_steps=1,
contexts=torch.tensor([[10, 11], [11, 12], [20, 21], [21, 22]], device="cuda"),
watermarking=torch.tensor([False, True], device="cuda"),
watermarking_skip_mask=torch.ones(4, dtype=torch.bool, device="cuda"),
watermarker=GumbelWatermarker(key=42, context_width=2),
)
torch.testing.assert_close(num_sampled, expected_num_sampled, rtol=0, atol=0)
for row, length in enumerate(num_sampled):
torch.testing.assert_close(
sampled[row, : int(length)], expected[row, : int(length)], rtol=0, atol=0
)
def test_watermarked_recovery_supports_smaller_draft_vocabulary():
target_logits = torch.tensor([float("-inf"), 0.0, 1.0, 2.0, 100.0], device="cuda")
draft_logits = torch.tensor([100.0, -100.0, -100.0, -100.0], device="cuda")
+62
View File
@@ -12,6 +12,7 @@ from vllm.v1.watermarking import (
derive_watermark_key,
)
from vllm.v1.watermarking.gumbel import _gamma_survival_integer_shape
from vllm.v1.watermarking.watermarker import RandomSampler
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
from vllm.v1.worker.gpu.sample.watermark import philox_gumbel_sample
@@ -197,3 +198,64 @@ def test_philox_gumbel_sample_skip_mask_matches_separate_samplers(use_fp64: bool
)
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
def test_philox_mixed_draft_sampling_caches_unprocessed_logits():
torch.manual_seed(0)
num_rows, vocab_size, num_steps = 5, 4099, 3
logits = torch.randn(num_rows, vocab_size, device="cuda", dtype=torch.float16)
temperatures = torch.tensor([0.0, 0.3, 0.5, 1.0, 2.0], device="cuda")
processed_logits = logits / torch.where(
temperatures == 0, 1, temperatures
).unsqueeze(-1)
contexts = torch.randint(0, vocab_size, (num_rows, 4), device="cuda")
idx_mapping = torch.arange(num_rows, dtype=torch.int32, device="cuda")
seeds = torch.arange(num_rows, dtype=torch.int64, device="cuda") + 10
positions = torch.arange(num_rows, dtype=torch.int64, device="cuda") + 20
# The skipped rows must divide by a temperature other than 1, or the
# unprocessed logits the cache stores are the logits the sampler sees.
# Row 0 keeps a greedy row in the skipped set.
skip_mask = torch.tensor([True, True, False, False, True], device="cuda")
cols = torch.arange(num_rows, dtype=torch.int32, device="cuda") % num_steps
cache = torch.zeros(
num_rows, num_steps, vocab_size + 1, dtype=logits.dtype, device="cuda"
)
ordinary = gumbel_sample(
logits,
idx_mapping,
temperatures,
seeds,
positions,
apply_temperature=True,
is_drafting=True,
)
watermarked = philox_gumbel_sample(processed_logits, contexts, 42)
expected = torch.where(skip_mask, ordinary, watermarked)
actual = (
GumbelWatermarker(key=42, context_width=4)
.sample(
processed_logits,
contexts,
random_sampler=RandomSampler(
expanded_idx_mapping=idx_mapping,
temperatures=temperatures,
seeds=seeds,
positions=positions,
is_drafting=True,
logits_cache=cache,
logits_cache_col=cols,
logits_cache_source=logits,
),
skip_mask=skip_mask,
)
.token_ids
)
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
stored = cache[torch.arange(num_rows, device="cuda"), cols.long(), :vocab_size]
assert torch.equal(stored.view(torch.int16), logits.view(torch.int16))
assert not cache[:, :, -1].any()
+550 -13
View File
@@ -28,6 +28,7 @@ from vllm.v1.watermarking.spec_decode import (
from vllm.v1.watermarking.watermarker import Watermarker, WatermarkSample
from vllm.v1.worker.gpu.sample.sampler import Sampler
from vllm.v1.worker.gpu.sample.watermark import (
draft_watermarking_mask,
philox_gumbel_sample,
repeated_context_mask,
)
@@ -202,11 +203,17 @@ def test_speculative_decoding_uses_fixed_dual_key_roles():
max_num_reqs=1,
device=torch.device("cpu"),
allow_target_only=False,
num_speculative_steps=3,
deduplicate_contexts="all",
deduplicate_contexts_max_history=None,
)
assert target.prf.key == derive_watermark_key(42, b"key_b")
assert draft is not None
assert draft.watermarker.prf.key == derive_watermark_key(42, b"key_a")
assert draft.num_speculative_steps == 3
assert draft.deduplicate_contexts == "all"
assert draft.deduplicate_contexts_max_history is None
def test_recovery_key_rejects_the_unsplit_dual_key_watermarker():
@@ -892,6 +899,187 @@ def test_gpu_sampler_builds_speculative_contexts_from_drafts():
assert torch.equal(contexts, torch.tensor([[10, 11], [11, 20], [20, 21]]))
def test_gpu_sampler_deduplicates_contexts_within_speculative_block():
sampler = object.__new__(GPUWatermarkSampler)
sampler.watermarker = SimpleNamespace(context_width=2)
sampler.deduplicate_contexts = "single_turn"
sampler.deduplicate_contexts_max_history = None
sampler.num_speculative_tokens = 4
sampler.req_states = SimpleNamespace(
all_token_ids=SimpleNamespace(gpu=torch.tensor([[100, 1, 2, 0, 0, 0]])),
prompt_len=SimpleNamespace(gpu=torch.tensor([1])),
total_len=SimpleNamespace(gpu=torch.tensor([3])),
)
request_indices = torch.tensor([0, 0, 0, 0, 0])
local_positions = torch.tensor([0, 1, 2, 3, 4])
contexts = sampler._get_contexts(
request_indices,
local_positions,
torch.tensor([0, 3, 4, 1, 2]),
)
repeated = sampler._get_repeated_contexts(
request_indices, contexts, local_positions
)
assert torch.equal(
contexts,
torch.tensor([[1, 2], [2, 3], [3, 4], [4, 1], [1, 2]]),
)
assert torch.equal(repeated, torch.tensor([False, False, False, False, True]))
sampler.deduplicate_contexts_max_history = 3
repeated = sampler._get_repeated_contexts(
request_indices, contexts, local_positions
)
assert not repeated.any()
def test_gpu_sampler_all_scope_skips_only_partial_speculative_contexts():
sampler = object.__new__(GPUWatermarkSampler)
sampler.watermarker = SimpleNamespace(context_width=2)
sampler.deduplicate_contexts = "all"
sampler.deduplicate_contexts_max_history = None
sampler.num_speculative_tokens = 3
sampler.req_states = SimpleNamespace(
all_token_ids=SimpleNamespace(gpu=torch.tensor([[100, 101, 10, 0, 0]])),
prompt_len=SimpleNamespace(gpu=torch.tensor([2])),
total_len=SimpleNamespace(gpu=torch.tensor([3])),
)
request_indices = torch.tensor([0, 0, 0])
local_positions = torch.tensor([0, 1, 2])
contexts = sampler._get_contexts(
request_indices,
local_positions,
torch.tensor([10, 20, 30]),
)
repeated = sampler._get_repeated_contexts(
request_indices, contexts, local_positions
)
assert torch.equal(contexts, torch.tensor([[-1, 10], [10, 20], [20, 30]]))
assert torch.equal(repeated, torch.tensor([True, False, False]))
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
@pytest.mark.parametrize("max_history,expected_last", [(None, True), (3, False)])
def test_speculative_context_repetition_accelerator_parity(
max_history: int | None, expected_last: bool
):
all_token_ids = torch.tensor([[10, 11]])
req_indices = torch.tensor([0, 0, 0, 0, 0])
prompt_lens = torch.tensor([0])
total_lens = torch.tensor([2])
local_positions = torch.tensor([0, 1, 2, 3, 4])
contexts = torch.tensor([[1, 2], [2, 3], [3, 4], [4, 1], [1, 2]])
expected = repeated_context_mask(
all_token_ids,
req_indices,
prompt_lens,
total_lens,
contexts,
history_offsets=local_positions,
local_positions=local_positions,
num_speculative_steps=4,
max_history=max_history,
)
actual = repeated_context_mask(
all_token_ids.cuda(),
req_indices.cuda(),
prompt_lens.cuda(),
total_lens.cuda(),
contexts.cuda(),
history_offsets=local_positions.cuda(),
local_positions=local_positions.cuda(),
num_speculative_steps=4,
max_history=max_history,
).cpu()
assert torch.equal(
expected, torch.tensor([False, False, False, False, expected_last])
)
assert torch.equal(actual, expected)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
@pytest.mark.parametrize("steps_layout", ["per_row", "broadcast"])
@pytest.mark.parametrize("req_indices_layout", ["contiguous", "column_view"])
def test_draft_context_deduplication_accelerator_parity(
steps_layout, req_indices_layout
):
"""Parity with the CPU reference for the layouts the speculators pass.
A draft step is one 0-d tensor broadcast over the batch, and DSpark samples
a step at a time, so its request indices are an `idx_map[:, i]` column. Both
are read per row, so a row that ignores the layout consults another row's
step or another request's history.
"""
num_rows, num_steps, step = 4, 3, 1
all_token_ids = torch.tensor([[10, 11, 20, 11], [30, 31, 11, 20]])
prompt_lens = torch.tensor([0, 0])
total_lens = torch.tensor([4, 4])
contexts = torch.tensor([[40, 41], [31, 11], [10, 11], [50, 51]])
prior_contexts = torch.zeros(num_rows, num_steps, 2, dtype=torch.int64)
prior_contexts[0, 0] = torch.tensor([40, 41])
enabled = torch.ones(num_rows, dtype=torch.bool)
req_indices = torch.tensor([0, 1, 0, 1])
def run(all_token_ids, req_indices, steps, prior):
return draft_watermarking_mask(
all_token_ids,
req_indices,
prompt_lens.to(all_token_ids.device),
total_lens.to(all_token_ids.device),
prior,
contexts.to(all_token_ids.device),
steps,
enabled.to(all_token_ids.device),
max_history=3,
include_prompt=False,
)
reference_prior = prior_contexts.clone()
reference = run(
all_token_ids,
req_indices,
torch.full((num_rows,), step),
reference_prior,
)
# Row 0 repeats its own step-0 context, rows 1 and 2 repeat a committed
# context of their own request, row 3 repeats nothing.
assert torch.equal(reference, torch.tensor([False, False, False, True]))
expected_prior = prior_contexts.clone()
expected_prior[:, step] = contexts
assert torch.equal(reference_prior, expected_prior)
if steps_layout == "per_row":
steps = torch.full((num_rows,), step).cuda()
else:
steps = torch.tensor(step).cuda().expand(num_rows)
assert steps.stride(0) == 0
if req_indices_layout == "contiguous":
gpu_req_indices = req_indices.cuda()
else:
gpu_req_indices = (
req_indices.repeat_interleave(num_steps).view(num_rows, num_steps).cuda()
)[:, 1]
assert gpu_req_indices.stride(0) == num_steps
actual_prior = prior_contexts.cuda()
actual = run(all_token_ids.cuda(), gpu_req_indices, steps, actual_prior).cpu()
assert torch.equal(actual, reference)
assert torch.equal(actual_prior.cpu(), expected_prior)
def test_gpu_sampler_builds_chunked_multi_request_speculative_contexts():
sampler = object.__new__(GPUWatermarkSampler)
sampler.watermarker = SimpleNamespace(context_width=2)
@@ -921,6 +1109,69 @@ def test_gpu_sampler_builds_chunked_multi_request_speculative_contexts():
)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
def test_draft_context_deduplication_broadcasts_the_draft_step():
"""The draft step is one 0-d tensor broadcast over the batch.
The context occurs four positions back, which `max_history - step` puts
just outside the scanned window, so a row that mistakes another row's step
for its own reports the opposite verdict. The neighbours leg is the
deterministic control: a row that reads a step of 0 instead of 2 widens its
window by two positions and finds the context. The production leg asserts
the layout the speculators pass, where the stride-less read lands outside
the allocation and can return the right value by chance.
"""
num_rows = 4
all_token_ids = torch.tensor([[3, 4, 9, 9, 9, 9], [3, 4, 9, 9, 9, 9]])
req_indices = torch.tensor([0, 0, 1, 1])
prompt_lens = torch.tensor([0, 0])
total_lens = torch.tensor([6, 6])
prior_contexts = torch.zeros(num_rows, 3, 2, dtype=torch.int64)
contexts = torch.tensor([[3, 4]]).expand(num_rows, 2).contiguous()
enabled = torch.ones(num_rows, dtype=torch.bool)
def run(steps, device):
prior = prior_contexts.clone().to(device)
active = draft_watermarking_mask(
all_token_ids.to(device),
req_indices.to(device),
prompt_lens.to(device),
total_lens.to(device),
prior,
contexts.to(device),
steps,
enabled.to(device),
max_history=4,
include_prompt=False,
)
return active.cpu(), prior.cpu()
expected, expected_prior = run(torch.full((num_rows,), 2), torch.device("cpu"))
assert torch.equal(expected, torch.ones(num_rows, dtype=torch.bool))
materialized, materialized_prior = run(
torch.full((num_rows,), 2).cuda(), torch.device("cuda")
)
assert torch.equal(materialized, expected)
assert torch.equal(materialized_prior, expected_prior)
# Neighbouring steps are the value a row reads when the broadcast stride is
# ignored; the production tensor reads uninitialized memory instead.
neighbours = torch.tensor([2, 0, 0, 0]).cuda()
broadcast, broadcast_prior = run(
neighbours[0].expand(num_rows), torch.device("cuda")
)
assert torch.equal(broadcast, expected)
assert torch.equal(broadcast_prior, expected_prior)
step = torch.tensor(2).cuda()
production, production_prior = run(step.expand(num_rows), torch.device("cuda"))
assert torch.equal(production, expected)
assert torch.equal(production_prior, expected_prior)
def test_draft_sampler_uses_draft_key_and_advances_context(monkeypatch):
class StubSpeculator(DraftModelSpeculator):
def capture(self): ...
@@ -937,22 +1188,32 @@ def test_draft_sampler_uses_draft_key_and_advances_context(monkeypatch):
return torch.zeros(hidden_states.shape[0], 8)
class StubWatermarker:
context_width = 2
@staticmethod
def sample(logits, contexts, random_sampler=None):
assert random_sampler is None
return WatermarkSample(torch.tensor([7, 7]), logits)
def sample(logits, contexts, random_sampler=None, skip_mask=None):
assert random_sampler is not None
return WatermarkSample(
torch.where(skip_mask, random_sampler(logits), 7), logits
)
speculator = object.__new__(StubSpeculator)
speculator.model = StubModel()
speculator.use_fp64_gumbel = False
speculator.acceptance_estimator = None
draft_watermarker = object.__new__(DraftWatermarker)
draft_watermarker.watermarker = StubWatermarker()
draft_watermarker.contexts = torch.tensor([[1, 2], [3, 4]])
draft_watermarker = DraftWatermarker(
StubWatermarker(),
max_num_reqs=2,
device=torch.device("cpu"),
num_speculative_steps=1,
deduplicate_contexts="none",
deduplicate_contexts_max_history=None,
)
draft_watermarker.contexts.copy_(torch.tensor([[1, 2], [3, 4]]))
draft_watermarker.enabled = torch.tensor([True, False])
speculator.draft_watermarker = draft_watermarker
monkeypatch.setattr(
"vllm.v1.worker.gpu.spec_decode.speculator.gumbel_sample",
"vllm.v1.watermarking.watermarker.gumbel_sample",
lambda *args, **kwargs: torch.tensor([3, 4]),
)
@@ -970,6 +1231,274 @@ def test_draft_sampler_uses_draft_key_and_advances_context(monkeypatch):
assert torch.equal(draft_watermarker.contexts, torch.tensor([[2, 7], [4, 4]]))
def test_draft_sampler_uses_ordinary_samples_for_repeated_contexts(monkeypatch):
class StubWatermarker:
context_width = 1
def __init__(self):
self.samples = iter((2, 1, 8))
def sample(self, logits, contexts, random_sampler=None, skip_mask=None):
watermarked = torch.tensor([next(self.samples)])
return WatermarkSample(
torch.where(skip_mask, random_sampler(logits), watermarked), logits
)
draft_watermarker = DraftWatermarker(
StubWatermarker(),
max_num_reqs=1,
device=torch.device("cpu"),
num_speculative_steps=3,
deduplicate_contexts="single_turn",
deduplicate_contexts_max_history=None,
)
draft_watermarker.prepare(
contexts=torch.tensor([[1]]),
enabled=torch.tensor([True]),
all_token_ids=torch.tensor([[100, 1, 0, 0]]),
prompt_lens=torch.tensor([1]),
total_lens=torch.tensor([2]),
)
idx_mapping = torch.tensor([0])
temperature = torch.tensor([1.0])
ordinary_samples = iter((torch.tensor([5]), torch.tensor([6]), torch.tensor([4])))
monkeypatch.setattr(
"vllm.v1.watermarking.watermarker.gumbel_sample",
lambda *args, **kwargs: next(ordinary_samples),
)
seeds = torch.tensor([0])
draft_logits = torch.zeros(1, 3, 8)
first = draft_watermarker.sample(
torch.zeros(1, 8),
idx_mapping,
temperature=temperature,
seed=seeds,
pos=torch.tensor([0]),
apply_temperature=True,
is_drafting=True,
logits_cache_col=torch.tensor(0),
logits_cache=draft_logits,
use_fp64=False,
)
second = draft_watermarker.sample(
torch.zeros(1, 8),
idx_mapping,
temperature=temperature,
seed=seeds,
pos=torch.tensor([1]),
apply_temperature=True,
is_drafting=True,
logits_cache_col=torch.tensor(1),
logits_cache=draft_logits,
use_fp64=False,
)
repeated = draft_watermarker.sample(
torch.zeros(1, 8),
idx_mapping,
temperature=temperature,
seed=seeds,
pos=torch.tensor([2]),
apply_temperature=True,
is_drafting=True,
logits_cache_col=torch.tensor(2),
logits_cache=draft_logits,
use_fp64=False,
)
assert first.item() == 2
assert second.item() == 1
assert repeated.item() == 4
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
def test_draft_context_deduplication_reads_strided_request_indices():
"""DSpark samples a step at a time, so req_indices is an `idx_map[:, i]` column."""
num_reqs = 4
num_speculative_steps = 3
all_token_ids = torch.tensor(
[[3, 4, 3, 4], [9, 9, 9, 9], [3, 4, 3, 4], [9, 9, 9, 9]]
).cuda()
prompt_lens = torch.zeros(num_reqs, dtype=torch.int64).cuda()
total_lens = torch.full((num_reqs,), 4).cuda()
contexts = torch.tensor([[3, 4]]).expand(num_reqs, 2).contiguous().cuda()
steps = torch.zeros(num_reqs, dtype=torch.int64).cuda()
enabled = torch.ones(num_reqs, dtype=torch.bool).cuda()
idx_map = (
torch.arange(num_reqs)
.repeat_interleave(num_speculative_steps)
.view(num_reqs, num_speculative_steps)
.cuda()
)
column = idx_map[:, 1]
assert column.stride(0) == num_speculative_steps
def run(req_indices):
prior = torch.zeros(
num_reqs, num_speculative_steps, 2, dtype=torch.int64, device="cuda"
)
return draft_watermarking_mask(
all_token_ids,
req_indices,
prompt_lens,
total_lens,
prior,
contexts,
steps,
enabled,
max_history=None,
include_prompt=False,
).cpu()
expected = torch.tensor([False, True, False, True])
assert torch.equal(run(column.contiguous()), expected)
assert torch.equal(run(column), expected)
def test_draft_sampler_deduplicates_against_committed_history(monkeypatch):
class StubWatermarker:
context_width = 1
@staticmethod
def sample(logits, contexts, random_sampler=None, skip_mask=None):
return WatermarkSample(
torch.where(skip_mask, random_sampler(logits), 7), logits
)
draft_watermarker = DraftWatermarker(
StubWatermarker(),
max_num_reqs=1,
device=torch.device("cpu"),
num_speculative_steps=1,
deduplicate_contexts="single_turn",
deduplicate_contexts_max_history=None,
)
draft_watermarker.prepare(
contexts=torch.tensor([[1]]),
enabled=torch.tensor([True]),
all_token_ids=torch.tensor([[100, 1, 2, 1]]),
prompt_lens=torch.tensor([1]),
total_lens=torch.tensor([4]),
)
monkeypatch.setattr(
"vllm.v1.watermarking.watermarker.gumbel_sample",
lambda *args, **kwargs: torch.tensor([4]),
)
sampled = draft_watermarker.sample(
torch.zeros(1, 8),
torch.tensor([0]),
temperature=torch.tensor([1.0]),
seed=torch.tensor([0]),
pos=torch.tensor([0]),
apply_temperature=True,
is_drafting=True,
logits_cache_col=torch.tensor(0),
logits_cache=torch.zeros(1, 1, 8),
use_fp64=False,
)
assert sampled.item() == 4
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
@pytest.mark.parametrize("idx_mapping_layout", ["contiguous", "column_view"])
def test_draft_watermarker_deduplicates_across_a_block_on_cuda(
monkeypatch, idx_mapping_layout
):
"""Drive the whole draft sampler over a block with the production shapes.
`draft_step` is a 0-d tensor, and a speculator that samples one step at a
time hands over a column of its index map. Each step is checked against the
committed history the window still reaches and against the earlier steps of
the block.
"""
num_reqs, num_steps, vocab_size = 2, 3, 16
watermarker = GumbelWatermarker(key=42, context_width=2)
draft_watermarker = DraftWatermarker(
watermarker,
max_num_reqs=num_reqs,
device=torch.device("cuda"),
num_speculative_steps=num_steps,
deduplicate_contexts="single_turn",
deduplicate_contexts_max_history=4,
)
draft_watermarker.prepare(
contexts=torch.tensor([[5, 6], [6, 3]]).cuda(),
enabled=torch.ones(num_reqs, dtype=torch.bool).cuda(),
all_token_ids=torch.tensor([[5, 6, 7, 5, 6], [8, 9, 6, 6, 3]]).cuda(),
prompt_lens=torch.zeros(num_reqs, dtype=torch.int64).cuda(),
total_lens=torch.full((num_reqs,), 5).cuda(),
)
if idx_mapping_layout == "contiguous":
idx_mapping = torch.arange(num_reqs, dtype=torch.int32).cuda()
else:
idx_mapping = (
torch.arange(num_reqs, dtype=torch.int32)
.repeat_interleave(num_steps)
.view(num_reqs, num_steps)
.cuda()
)[:, 1]
assert idx_mapping.stride(0) == num_steps
skip_masks: list[torch.Tensor] = []
original = philox_gumbel_sample
def spy(*args, **kwargs):
skip_masks.append(kwargs["skip_mask"].cpu())
return original(*args, **kwargs)
monkeypatch.setattr("vllm.v1.worker.gpu.sample.watermark.philox_gumbel_sample", spy)
draft_logits = torch.zeros(num_reqs, num_steps, vocab_size + 1).cuda()
step_logits = []
for step in range(num_steps):
# Token 6 wins by a margin no Gumbel draw can close, on either path.
logits = torch.zeros(num_reqs, vocab_size).cuda()
logits[:, 6] = 100.0
logits[:, 0] = float(step) + 1.0
step_logits.append(logits)
sampled = draft_watermarker.sample(
logits,
idx_mapping,
temperature=torch.ones(num_reqs).cuda(),
seed=torch.zeros(num_reqs, dtype=torch.int64).cuda(),
pos=torch.full((num_reqs,), 5 + step, dtype=torch.int64).cuda(),
apply_temperature=True,
is_drafting=True,
logits_cache_col=torch.tensor(step).cuda(),
logits_cache=draft_logits,
use_fp64=False,
)
assert torch.equal(sampled.cpu(), torch.tensor([6, 6]))
# Step 0: request 0's [5, 6] is committed at position 2, request 1's [6, 3]
# is new. Step 1: [6, 6] and [3, 6] are both new. Step 2: request 0's
# [6, 6] repeats its own step-1 context and request 1's repeats position 4.
assert [mask.tolist() for mask in skip_masks] == [
[True, False],
[False, False],
[True, True],
]
assert torch.equal(
draft_watermarker.prior_contexts[:num_reqs].cpu(),
torch.tensor([[[5, 6], [6, 6], [6, 6]], [[6, 3], [3, 6], [6, 6]]]),
)
assert torch.equal(
draft_watermarker.contexts[:num_reqs].cpu(), torch.tensor([[6, 6], [6, 6]])
)
for step, logits in enumerate(step_logits):
# The 0-d draft step selects the logits cache column.
assert torch.equal(draft_logits[:, step, :vocab_size], logits)
assert not draft_logits[:, :, -1].any()
def test_dspark_reduced_vocab_draft_sampler_applies_watermarking(monkeypatch):
speculator = object.__new__(DSparkSpeculator)
speculator.draft_logits = torch.zeros(2, 1, 8)
@@ -982,15 +1511,23 @@ def test_dspark_reduced_vocab_draft_sampler_applies_watermarking(monkeypatch):
speculator.acceptance_estimator = None
watermark_logits: list[torch.Tensor] = []
def sample(logits, sampled, idx_map, temperature):
def sample(
logits,
idx_mapping,
temperature,
seed,
pos,
apply_temperature,
is_drafting,
logits_cache=None,
logits_cache_col=None,
use_fp64=False,
logits_cache_source=None,
):
watermark_logits.append(logits.clone())
return sampled + 1
return torch.tensor([4, 5])
speculator.draft_watermarker = SimpleNamespace(sample=sample)
monkeypatch.setattr(
"vllm.v1.worker.gpu.spec_decode.dspark.speculator.gumbel_sample",
lambda *args, **kwargs: torch.tensor([3, 4]),
)
sampled = speculator._sample_logits(
torch.tensor([[10.0, 20.0], [30.0, 40.0]]),