[watermarking] add context deduplication support to speculative decoding (#56807)
Signed-off-by: Raphael Rialland <raphael.rialland@mistral.ai> Signed-off-by: Simon Veitner <sveitner@redhat.com> Co-authored-by: Simon Veitner <sveitner@redhat.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Simon Veitner
Lucas Wilkinson
parent
73a5831127
commit
0f8b398158
@@ -77,30 +77,17 @@ watermarked. The watermark signal is diluted in proportion to the share of
|
||||
output tokens supplied by accepted drafts; rejected drafts do not dilute it
|
||||
because their recovery tokens are watermarked.
|
||||
|
||||
Speculative-decoding token paths do not currently support generation-side
|
||||
context deduplication. The configured `deduplicate_contexts` policy is not
|
||||
applied to accepted drafts, rejection-recovery tokens, or bonus tokens.
|
||||
|
||||
For `dual_key_gumbel`, `alpha` has no effect under speculative decoding. The
|
||||
speculative protocol selects the key for each token instead.
|
||||
|
||||
## Algorithms
|
||||
|
||||
### Gumbel-max
|
||||
|
||||
Gumbel-max derives a deterministic pseudorandom value from the key, prior token
|
||||
context, and every candidate token, then uses the resulting Gumbel noise for
|
||||
categorical sampling. See
|
||||
[Aaronson's original presentation](https://simons.berkeley.edu/sites/default/files/2024-10/LLM24-2%20Slides%20-%20Scott%20Aaronson.pdf).
|
||||
|
||||
Gumbel-max requires stochastic sampling. Greedy requests (`temperature=0`)
|
||||
bypass watermarking and emit a warning once per worker.
|
||||
## Context deduplication
|
||||
|
||||
When a token context is repeated, generation can use ordinary sampling for that
|
||||
occurrence. Reusing the repeated context leads to a bias over the sequence, as
|
||||
certain token choices would be correlated. Ordinary sampling at these positions
|
||||
allows for single-sequence non-distortion (see section G.3 of the
|
||||
[SynthID-Text supplementary materials](https://media.springernature.com/original/springer-static/esm/art%3A10.1038%2Fs41586-024-08025-4/MediaObjects/41586_2024_8025_MOESM1_ESM.pdf)).
|
||||
This also applies to drafter watermarking during speculative decoding.
|
||||
The detector independently deduplicates contexts so repeated keyed random
|
||||
vectors are not treated as independent evidence, meaning that context
|
||||
deduplication at generation time does not reduce the watermarking signal, unless
|
||||
@@ -143,6 +130,18 @@ vllm serve MODEL \
|
||||
'{"algorithm":"gumbel","key":42,"deduplicate_contexts":"all"}'
|
||||
```
|
||||
|
||||
## Algorithms
|
||||
|
||||
### Gumbel-max
|
||||
|
||||
Gumbel-max derives a deterministic pseudorandom value from the key, prior token
|
||||
context, and every candidate token, then uses the resulting Gumbel noise for
|
||||
categorical sampling. See
|
||||
[Aaronson's original presentation](https://simons.berkeley.edu/sites/default/files/2024-10/LLM24-2%20Slides%20-%20Scott%20Aaronson.pdf).
|
||||
|
||||
Gumbel-max requires stochastic sampling. Greedy requests (`temperature=0`)
|
||||
bypass watermarking and emit a warning once per worker.
|
||||
|
||||
### Dual-key Gumbel-max
|
||||
|
||||
Dual-key Gumbel-max derives independent keys A and B from one configured master
|
||||
|
||||
+23
-2
@@ -3526,7 +3526,6 @@ def test_target_only_gumbel_allows_speculative_decoding(caplog_vllm, disable_log
|
||||
config._check_watermarking_unsupported()
|
||||
|
||||
assert "Target-only watermarking leaves accepted draft tokens" in caplog_vllm.text
|
||||
assert "Context deduplication is not supported" in caplog_vllm.text
|
||||
|
||||
|
||||
def test_speculative_watermarking_without_context_dedup_does_not_warn(
|
||||
@@ -3543,10 +3542,32 @@ def test_speculative_watermarking_without_context_dedup_does_not_warn(
|
||||
parallel_drafting=False,
|
||||
)
|
||||
|
||||
caplog_vllm.clear()
|
||||
with caplog_vllm.at_level(logging.WARNING):
|
||||
config._check_watermarking_unsupported()
|
||||
|
||||
assert "Context deduplication is not supported" not in caplog_vllm.text
|
||||
assert "dedup" not in caplog_vllm.text.lower()
|
||||
|
||||
|
||||
def test_speculative_context_dedup_logs_no_unsupported_warning(
|
||||
caplog_vllm, disable_log_dedup
|
||||
):
|
||||
config = _watermarked_vllm_config()
|
||||
config.watermark_config = WatermarkConfig(
|
||||
algorithm="dual_key_gumbel", key=42, deduplicate_contexts="single_turn"
|
||||
)
|
||||
config.speculative_config = SimpleNamespace(
|
||||
method="mtp",
|
||||
draft_sample_method="probabilistic",
|
||||
rejection_sample_method="standard",
|
||||
parallel_drafting=False,
|
||||
)
|
||||
|
||||
caplog_vllm.clear()
|
||||
with caplog_vllm.at_level(logging.WARNING):
|
||||
config._check_watermarking_unsupported()
|
||||
|
||||
assert "dedup" not in caplog_vllm.text.lower()
|
||||
|
||||
|
||||
def test_gumbel_rejects_speculative_decoding_without_target_only():
|
||||
|
||||
@@ -203,6 +203,182 @@ def test_watermarked_recovery_uses_target_key():
|
||||
assert sampled[0, 0] == expected.token_ids[0]
|
||||
|
||||
|
||||
def test_repeated_draft_context_does_not_change_acceptance():
|
||||
torch.manual_seed(0)
|
||||
num_trials = 8
|
||||
# A draft distribution of its own, so the drafts are not accepted outright.
|
||||
target_logits = torch.randn(64, device="cuda")
|
||||
draft_logits = torch.randn(64, device="cuda")
|
||||
inputs = _build_rejection_sample_inputs(
|
||||
target_logits,
|
||||
draft_logits,
|
||||
num_speculative_steps=1,
|
||||
temperature=1.0,
|
||||
num_trials=num_trials,
|
||||
)
|
||||
expected, expected_num_sampled = rejection_sample(**inputs, num_speculative_steps=1)
|
||||
|
||||
sampled, num_sampled = watermarked_rejection_sample(
|
||||
**inputs,
|
||||
num_speculative_steps=1,
|
||||
contexts=torch.tensor([[10, 11], [11, 12]], device="cuda").repeat(
|
||||
num_trials, 1
|
||||
),
|
||||
watermarking=torch.ones(num_trials, dtype=torch.bool, device="cuda"),
|
||||
watermarking_skip_mask=torch.tensor([True, False], device="cuda").repeat(
|
||||
num_trials
|
||||
),
|
||||
watermarker=GumbelWatermarker(key=42, context_width=2),
|
||||
)
|
||||
|
||||
# Some drafts must actually be rejected, or the ordinary-residual branch
|
||||
# the skip mask selects is never reached.
|
||||
assert (expected_num_sampled == 1).any()
|
||||
torch.testing.assert_close(num_sampled, expected_num_sampled, rtol=0, atol=0)
|
||||
torch.testing.assert_close(sampled[:, :1], expected[:, :1], rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_repeated_context_skip_mask_is_read_per_row_in_a_block():
|
||||
"""A block of three drafts, accepted at row 0 and rejected at row 1.
|
||||
|
||||
The skip mask selects the sampler per row, so the row the target recovers
|
||||
at decides which key draws the recovery token.
|
||||
"""
|
||||
num_speculative_steps = 3
|
||||
target_logits = torch.tensor([float("-inf"), 0.0, 1.0, 2.0], device="cuda")
|
||||
draft_logits = torch.tensor([30.0, -30.0, -30.0, -30.0], device="cuda")
|
||||
inputs = _build_rejection_sample_inputs(
|
||||
target_logits,
|
||||
draft_logits,
|
||||
num_speculative_steps=num_speculative_steps,
|
||||
temperature=1.0,
|
||||
num_trials=1,
|
||||
)
|
||||
# Row 0 drafts a token the target gives 0.67 and the draft e**-60, so it is
|
||||
# accepted; row 1 drafts a token the target gives no mass, so it is not.
|
||||
inputs["draft_sampled"][1] = 3
|
||||
inputs["draft_sampled"][2] = 0
|
||||
contexts = torch.tensor(
|
||||
[[10, 11], [11, 3], [3, 0], [0, 1]], dtype=torch.int64, device="cuda"
|
||||
)
|
||||
expected, expected_num_sampled = rejection_sample(
|
||||
**inputs, num_speculative_steps=num_speculative_steps
|
||||
)
|
||||
assert expected_num_sampled.item() == 2
|
||||
assert expected[0, 0].item() == 3
|
||||
|
||||
watermarker = GumbelWatermarker(key=42, context_width=2)
|
||||
skipped, skipped_num_sampled = watermarked_rejection_sample(
|
||||
**inputs,
|
||||
num_speculative_steps=num_speculative_steps,
|
||||
contexts=contexts,
|
||||
watermarking=torch.tensor([True], device="cuda"),
|
||||
watermarking_skip_mask=torch.tensor([False, True, False, False], device="cuda"),
|
||||
watermarker=watermarker,
|
||||
)
|
||||
|
||||
assert skipped_num_sampled.item() == 2
|
||||
torch.testing.assert_close(skipped[:, :2], expected[:, :2], rtol=0, atol=0)
|
||||
|
||||
watermarked, watermarked_num_sampled = watermarked_rejection_sample(
|
||||
**inputs,
|
||||
num_speculative_steps=num_speculative_steps,
|
||||
contexts=contexts,
|
||||
watermarking=torch.tensor([True], device="cuda"),
|
||||
watermarking_skip_mask=torch.zeros(
|
||||
num_speculative_steps + 1, dtype=torch.bool, device="cuda"
|
||||
),
|
||||
watermarker=watermarker,
|
||||
)
|
||||
# The residual equals the target to within e**-60, so the recovery token is
|
||||
# the target key's draw on the rejected row's context.
|
||||
recovery = GumbelWatermarker(key=42, context_width=2).sample(
|
||||
target_logits.unsqueeze(0), contexts[1:2], lambda _: None
|
||||
)
|
||||
|
||||
assert watermarked_num_sampled.item() == 2
|
||||
assert watermarked[0, 0].item() == 3
|
||||
assert watermarked[0, 1] == recovery.token_ids[0]
|
||||
|
||||
|
||||
def test_repeated_recovery_context_uses_ordinary_residual_sample():
|
||||
target_logits = torch.tensor([float("-inf"), 0.0, 1.0, 2.0], device="cuda")
|
||||
draft_logits = torch.tensor([100.0, -100.0, -100.0, -100.0], device="cuda")
|
||||
inputs = _build_rejection_sample_inputs(
|
||||
target_logits,
|
||||
draft_logits,
|
||||
num_speculative_steps=1,
|
||||
temperature=1.0,
|
||||
num_trials=1,
|
||||
)
|
||||
inputs["draft_sampled"][1] = 0
|
||||
expected, expected_num_sampled = rejection_sample(**inputs, num_speculative_steps=1)
|
||||
|
||||
sampled, num_sampled = watermarked_rejection_sample(
|
||||
**inputs,
|
||||
num_speculative_steps=1,
|
||||
contexts=torch.tensor([[10, 11], [11, 0]], device="cuda"),
|
||||
watermarking=torch.tensor([True], device="cuda"),
|
||||
watermarking_skip_mask=torch.tensor([True, False], device="cuda"),
|
||||
watermarker=GumbelWatermarker(key=42, context_width=2),
|
||||
)
|
||||
|
||||
torch.testing.assert_close(num_sampled, expected_num_sampled, rtol=0, atol=0)
|
||||
torch.testing.assert_close(sampled[:, :1], expected[:, :1], rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_repeated_bonus_context_uses_ordinary_target_sample():
|
||||
target_logits = torch.randn(64, device="cuda")
|
||||
inputs = _build_rejection_sample_inputs(
|
||||
target_logits,
|
||||
target_logits,
|
||||
num_speculative_steps=1,
|
||||
temperature=1.0,
|
||||
num_trials=1,
|
||||
)
|
||||
expected, expected_num_sampled = rejection_sample(**inputs, num_speculative_steps=1)
|
||||
|
||||
sampled, num_sampled = watermarked_rejection_sample(
|
||||
**inputs,
|
||||
num_speculative_steps=1,
|
||||
contexts=torch.tensor([[10, 11], [11, 12]], device="cuda"),
|
||||
watermarking=torch.tensor([True], device="cuda"),
|
||||
watermarking_skip_mask=torch.tensor([False, True], device="cuda"),
|
||||
watermarker=GumbelWatermarker(key=42, context_width=2),
|
||||
)
|
||||
|
||||
torch.testing.assert_close(num_sampled, expected_num_sampled, rtol=0, atol=0)
|
||||
torch.testing.assert_close(sampled[:, :2], expected[:, :2], rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_repeated_context_does_not_change_unwatermarked_or_greedy_rows():
|
||||
target_logits = torch.randn(64, device="cuda")
|
||||
inputs = _build_rejection_sample_inputs(
|
||||
target_logits,
|
||||
target_logits,
|
||||
num_speculative_steps=1,
|
||||
temperature=1.0,
|
||||
num_trials=2,
|
||||
)
|
||||
inputs["temperature"][1] = 0
|
||||
expected, expected_num_sampled = rejection_sample(**inputs, num_speculative_steps=1)
|
||||
|
||||
sampled, num_sampled = watermarked_rejection_sample(
|
||||
**inputs,
|
||||
num_speculative_steps=1,
|
||||
contexts=torch.tensor([[10, 11], [11, 12], [20, 21], [21, 22]], device="cuda"),
|
||||
watermarking=torch.tensor([False, True], device="cuda"),
|
||||
watermarking_skip_mask=torch.ones(4, dtype=torch.bool, device="cuda"),
|
||||
watermarker=GumbelWatermarker(key=42, context_width=2),
|
||||
)
|
||||
|
||||
torch.testing.assert_close(num_sampled, expected_num_sampled, rtol=0, atol=0)
|
||||
for row, length in enumerate(num_sampled):
|
||||
torch.testing.assert_close(
|
||||
sampled[row, : int(length)], expected[row, : int(length)], rtol=0, atol=0
|
||||
)
|
||||
|
||||
|
||||
def test_watermarked_recovery_supports_smaller_draft_vocabulary():
|
||||
target_logits = torch.tensor([float("-inf"), 0.0, 1.0, 2.0, 100.0], device="cuda")
|
||||
draft_logits = torch.tensor([100.0, -100.0, -100.0, -100.0], device="cuda")
|
||||
|
||||
@@ -12,6 +12,7 @@ from vllm.v1.watermarking import (
|
||||
derive_watermark_key,
|
||||
)
|
||||
from vllm.v1.watermarking.gumbel import _gamma_survival_integer_shape
|
||||
from vllm.v1.watermarking.watermarker import RandomSampler
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
|
||||
from vllm.v1.worker.gpu.sample.watermark import philox_gumbel_sample
|
||||
|
||||
@@ -197,3 +198,64 @@ def test_philox_gumbel_sample_skip_mask_matches_separate_samplers(use_fp64: bool
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
|
||||
)
|
||||
def test_philox_mixed_draft_sampling_caches_unprocessed_logits():
|
||||
torch.manual_seed(0)
|
||||
num_rows, vocab_size, num_steps = 5, 4099, 3
|
||||
logits = torch.randn(num_rows, vocab_size, device="cuda", dtype=torch.float16)
|
||||
temperatures = torch.tensor([0.0, 0.3, 0.5, 1.0, 2.0], device="cuda")
|
||||
processed_logits = logits / torch.where(
|
||||
temperatures == 0, 1, temperatures
|
||||
).unsqueeze(-1)
|
||||
contexts = torch.randint(0, vocab_size, (num_rows, 4), device="cuda")
|
||||
idx_mapping = torch.arange(num_rows, dtype=torch.int32, device="cuda")
|
||||
seeds = torch.arange(num_rows, dtype=torch.int64, device="cuda") + 10
|
||||
positions = torch.arange(num_rows, dtype=torch.int64, device="cuda") + 20
|
||||
# The skipped rows must divide by a temperature other than 1, or the
|
||||
# unprocessed logits the cache stores are the logits the sampler sees.
|
||||
# Row 0 keeps a greedy row in the skipped set.
|
||||
skip_mask = torch.tensor([True, True, False, False, True], device="cuda")
|
||||
cols = torch.arange(num_rows, dtype=torch.int32, device="cuda") % num_steps
|
||||
cache = torch.zeros(
|
||||
num_rows, num_steps, vocab_size + 1, dtype=logits.dtype, device="cuda"
|
||||
)
|
||||
|
||||
ordinary = gumbel_sample(
|
||||
logits,
|
||||
idx_mapping,
|
||||
temperatures,
|
||||
seeds,
|
||||
positions,
|
||||
apply_temperature=True,
|
||||
is_drafting=True,
|
||||
)
|
||||
watermarked = philox_gumbel_sample(processed_logits, contexts, 42)
|
||||
expected = torch.where(skip_mask, ordinary, watermarked)
|
||||
actual = (
|
||||
GumbelWatermarker(key=42, context_width=4)
|
||||
.sample(
|
||||
processed_logits,
|
||||
contexts,
|
||||
random_sampler=RandomSampler(
|
||||
expanded_idx_mapping=idx_mapping,
|
||||
temperatures=temperatures,
|
||||
seeds=seeds,
|
||||
positions=positions,
|
||||
is_drafting=True,
|
||||
logits_cache=cache,
|
||||
logits_cache_col=cols,
|
||||
logits_cache_source=logits,
|
||||
),
|
||||
skip_mask=skip_mask,
|
||||
)
|
||||
.token_ids
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
stored = cache[torch.arange(num_rows, device="cuda"), cols.long(), :vocab_size]
|
||||
assert torch.equal(stored.view(torch.int16), logits.view(torch.int16))
|
||||
assert not cache[:, :, -1].any()
|
||||
|
||||
@@ -28,6 +28,7 @@ from vllm.v1.watermarking.spec_decode import (
|
||||
from vllm.v1.watermarking.watermarker import Watermarker, WatermarkSample
|
||||
from vllm.v1.worker.gpu.sample.sampler import Sampler
|
||||
from vllm.v1.worker.gpu.sample.watermark import (
|
||||
draft_watermarking_mask,
|
||||
philox_gumbel_sample,
|
||||
repeated_context_mask,
|
||||
)
|
||||
@@ -202,11 +203,17 @@ def test_speculative_decoding_uses_fixed_dual_key_roles():
|
||||
max_num_reqs=1,
|
||||
device=torch.device("cpu"),
|
||||
allow_target_only=False,
|
||||
num_speculative_steps=3,
|
||||
deduplicate_contexts="all",
|
||||
deduplicate_contexts_max_history=None,
|
||||
)
|
||||
|
||||
assert target.prf.key == derive_watermark_key(42, b"key_b")
|
||||
assert draft is not None
|
||||
assert draft.watermarker.prf.key == derive_watermark_key(42, b"key_a")
|
||||
assert draft.num_speculative_steps == 3
|
||||
assert draft.deduplicate_contexts == "all"
|
||||
assert draft.deduplicate_contexts_max_history is None
|
||||
|
||||
|
||||
def test_recovery_key_rejects_the_unsplit_dual_key_watermarker():
|
||||
@@ -892,6 +899,187 @@ def test_gpu_sampler_builds_speculative_contexts_from_drafts():
|
||||
assert torch.equal(contexts, torch.tensor([[10, 11], [11, 20], [20, 21]]))
|
||||
|
||||
|
||||
def test_gpu_sampler_deduplicates_contexts_within_speculative_block():
|
||||
sampler = object.__new__(GPUWatermarkSampler)
|
||||
sampler.watermarker = SimpleNamespace(context_width=2)
|
||||
sampler.deduplicate_contexts = "single_turn"
|
||||
sampler.deduplicate_contexts_max_history = None
|
||||
sampler.num_speculative_tokens = 4
|
||||
sampler.req_states = SimpleNamespace(
|
||||
all_token_ids=SimpleNamespace(gpu=torch.tensor([[100, 1, 2, 0, 0, 0]])),
|
||||
prompt_len=SimpleNamespace(gpu=torch.tensor([1])),
|
||||
total_len=SimpleNamespace(gpu=torch.tensor([3])),
|
||||
)
|
||||
request_indices = torch.tensor([0, 0, 0, 0, 0])
|
||||
local_positions = torch.tensor([0, 1, 2, 3, 4])
|
||||
contexts = sampler._get_contexts(
|
||||
request_indices,
|
||||
local_positions,
|
||||
torch.tensor([0, 3, 4, 1, 2]),
|
||||
)
|
||||
|
||||
repeated = sampler._get_repeated_contexts(
|
||||
request_indices, contexts, local_positions
|
||||
)
|
||||
|
||||
assert torch.equal(
|
||||
contexts,
|
||||
torch.tensor([[1, 2], [2, 3], [3, 4], [4, 1], [1, 2]]),
|
||||
)
|
||||
assert torch.equal(repeated, torch.tensor([False, False, False, False, True]))
|
||||
|
||||
sampler.deduplicate_contexts_max_history = 3
|
||||
repeated = sampler._get_repeated_contexts(
|
||||
request_indices, contexts, local_positions
|
||||
)
|
||||
assert not repeated.any()
|
||||
|
||||
|
||||
def test_gpu_sampler_all_scope_skips_only_partial_speculative_contexts():
|
||||
sampler = object.__new__(GPUWatermarkSampler)
|
||||
sampler.watermarker = SimpleNamespace(context_width=2)
|
||||
sampler.deduplicate_contexts = "all"
|
||||
sampler.deduplicate_contexts_max_history = None
|
||||
sampler.num_speculative_tokens = 3
|
||||
sampler.req_states = SimpleNamespace(
|
||||
all_token_ids=SimpleNamespace(gpu=torch.tensor([[100, 101, 10, 0, 0]])),
|
||||
prompt_len=SimpleNamespace(gpu=torch.tensor([2])),
|
||||
total_len=SimpleNamespace(gpu=torch.tensor([3])),
|
||||
)
|
||||
request_indices = torch.tensor([0, 0, 0])
|
||||
local_positions = torch.tensor([0, 1, 2])
|
||||
contexts = sampler._get_contexts(
|
||||
request_indices,
|
||||
local_positions,
|
||||
torch.tensor([10, 20, 30]),
|
||||
)
|
||||
|
||||
repeated = sampler._get_repeated_contexts(
|
||||
request_indices, contexts, local_positions
|
||||
)
|
||||
|
||||
assert torch.equal(contexts, torch.tensor([[-1, 10], [10, 20], [20, 30]]))
|
||||
assert torch.equal(repeated, torch.tensor([True, False, False]))
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
|
||||
)
|
||||
@pytest.mark.parametrize("max_history,expected_last", [(None, True), (3, False)])
|
||||
def test_speculative_context_repetition_accelerator_parity(
|
||||
max_history: int | None, expected_last: bool
|
||||
):
|
||||
all_token_ids = torch.tensor([[10, 11]])
|
||||
req_indices = torch.tensor([0, 0, 0, 0, 0])
|
||||
prompt_lens = torch.tensor([0])
|
||||
total_lens = torch.tensor([2])
|
||||
local_positions = torch.tensor([0, 1, 2, 3, 4])
|
||||
contexts = torch.tensor([[1, 2], [2, 3], [3, 4], [4, 1], [1, 2]])
|
||||
|
||||
expected = repeated_context_mask(
|
||||
all_token_ids,
|
||||
req_indices,
|
||||
prompt_lens,
|
||||
total_lens,
|
||||
contexts,
|
||||
history_offsets=local_positions,
|
||||
local_positions=local_positions,
|
||||
num_speculative_steps=4,
|
||||
max_history=max_history,
|
||||
)
|
||||
actual = repeated_context_mask(
|
||||
all_token_ids.cuda(),
|
||||
req_indices.cuda(),
|
||||
prompt_lens.cuda(),
|
||||
total_lens.cuda(),
|
||||
contexts.cuda(),
|
||||
history_offsets=local_positions.cuda(),
|
||||
local_positions=local_positions.cuda(),
|
||||
num_speculative_steps=4,
|
||||
max_history=max_history,
|
||||
).cpu()
|
||||
|
||||
assert torch.equal(
|
||||
expected, torch.tensor([False, False, False, False, expected_last])
|
||||
)
|
||||
assert torch.equal(actual, expected)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
|
||||
)
|
||||
@pytest.mark.parametrize("steps_layout", ["per_row", "broadcast"])
|
||||
@pytest.mark.parametrize("req_indices_layout", ["contiguous", "column_view"])
|
||||
def test_draft_context_deduplication_accelerator_parity(
|
||||
steps_layout, req_indices_layout
|
||||
):
|
||||
"""Parity with the CPU reference for the layouts the speculators pass.
|
||||
|
||||
A draft step is one 0-d tensor broadcast over the batch, and DSpark samples
|
||||
a step at a time, so its request indices are an `idx_map[:, i]` column. Both
|
||||
are read per row, so a row that ignores the layout consults another row's
|
||||
step or another request's history.
|
||||
"""
|
||||
num_rows, num_steps, step = 4, 3, 1
|
||||
all_token_ids = torch.tensor([[10, 11, 20, 11], [30, 31, 11, 20]])
|
||||
prompt_lens = torch.tensor([0, 0])
|
||||
total_lens = torch.tensor([4, 4])
|
||||
contexts = torch.tensor([[40, 41], [31, 11], [10, 11], [50, 51]])
|
||||
prior_contexts = torch.zeros(num_rows, num_steps, 2, dtype=torch.int64)
|
||||
prior_contexts[0, 0] = torch.tensor([40, 41])
|
||||
enabled = torch.ones(num_rows, dtype=torch.bool)
|
||||
req_indices = torch.tensor([0, 1, 0, 1])
|
||||
|
||||
def run(all_token_ids, req_indices, steps, prior):
|
||||
return draft_watermarking_mask(
|
||||
all_token_ids,
|
||||
req_indices,
|
||||
prompt_lens.to(all_token_ids.device),
|
||||
total_lens.to(all_token_ids.device),
|
||||
prior,
|
||||
contexts.to(all_token_ids.device),
|
||||
steps,
|
||||
enabled.to(all_token_ids.device),
|
||||
max_history=3,
|
||||
include_prompt=False,
|
||||
)
|
||||
|
||||
reference_prior = prior_contexts.clone()
|
||||
reference = run(
|
||||
all_token_ids,
|
||||
req_indices,
|
||||
torch.full((num_rows,), step),
|
||||
reference_prior,
|
||||
)
|
||||
|
||||
# Row 0 repeats its own step-0 context, rows 1 and 2 repeat a committed
|
||||
# context of their own request, row 3 repeats nothing.
|
||||
assert torch.equal(reference, torch.tensor([False, False, False, True]))
|
||||
expected_prior = prior_contexts.clone()
|
||||
expected_prior[:, step] = contexts
|
||||
assert torch.equal(reference_prior, expected_prior)
|
||||
|
||||
if steps_layout == "per_row":
|
||||
steps = torch.full((num_rows,), step).cuda()
|
||||
else:
|
||||
steps = torch.tensor(step).cuda().expand(num_rows)
|
||||
assert steps.stride(0) == 0
|
||||
|
||||
if req_indices_layout == "contiguous":
|
||||
gpu_req_indices = req_indices.cuda()
|
||||
else:
|
||||
gpu_req_indices = (
|
||||
req_indices.repeat_interleave(num_steps).view(num_rows, num_steps).cuda()
|
||||
)[:, 1]
|
||||
assert gpu_req_indices.stride(0) == num_steps
|
||||
|
||||
actual_prior = prior_contexts.cuda()
|
||||
actual = run(all_token_ids.cuda(), gpu_req_indices, steps, actual_prior).cpu()
|
||||
|
||||
assert torch.equal(actual, reference)
|
||||
assert torch.equal(actual_prior.cpu(), expected_prior)
|
||||
|
||||
|
||||
def test_gpu_sampler_builds_chunked_multi_request_speculative_contexts():
|
||||
sampler = object.__new__(GPUWatermarkSampler)
|
||||
sampler.watermarker = SimpleNamespace(context_width=2)
|
||||
@@ -921,6 +1109,69 @@ def test_gpu_sampler_builds_chunked_multi_request_speculative_contexts():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
|
||||
)
|
||||
def test_draft_context_deduplication_broadcasts_the_draft_step():
|
||||
"""The draft step is one 0-d tensor broadcast over the batch.
|
||||
|
||||
The context occurs four positions back, which `max_history - step` puts
|
||||
just outside the scanned window, so a row that mistakes another row's step
|
||||
for its own reports the opposite verdict. The neighbours leg is the
|
||||
deterministic control: a row that reads a step of 0 instead of 2 widens its
|
||||
window by two positions and finds the context. The production leg asserts
|
||||
the layout the speculators pass, where the stride-less read lands outside
|
||||
the allocation and can return the right value by chance.
|
||||
"""
|
||||
num_rows = 4
|
||||
all_token_ids = torch.tensor([[3, 4, 9, 9, 9, 9], [3, 4, 9, 9, 9, 9]])
|
||||
req_indices = torch.tensor([0, 0, 1, 1])
|
||||
prompt_lens = torch.tensor([0, 0])
|
||||
total_lens = torch.tensor([6, 6])
|
||||
prior_contexts = torch.zeros(num_rows, 3, 2, dtype=torch.int64)
|
||||
contexts = torch.tensor([[3, 4]]).expand(num_rows, 2).contiguous()
|
||||
enabled = torch.ones(num_rows, dtype=torch.bool)
|
||||
|
||||
def run(steps, device):
|
||||
prior = prior_contexts.clone().to(device)
|
||||
active = draft_watermarking_mask(
|
||||
all_token_ids.to(device),
|
||||
req_indices.to(device),
|
||||
prompt_lens.to(device),
|
||||
total_lens.to(device),
|
||||
prior,
|
||||
contexts.to(device),
|
||||
steps,
|
||||
enabled.to(device),
|
||||
max_history=4,
|
||||
include_prompt=False,
|
||||
)
|
||||
return active.cpu(), prior.cpu()
|
||||
|
||||
expected, expected_prior = run(torch.full((num_rows,), 2), torch.device("cpu"))
|
||||
assert torch.equal(expected, torch.ones(num_rows, dtype=torch.bool))
|
||||
|
||||
materialized, materialized_prior = run(
|
||||
torch.full((num_rows,), 2).cuda(), torch.device("cuda")
|
||||
)
|
||||
assert torch.equal(materialized, expected)
|
||||
assert torch.equal(materialized_prior, expected_prior)
|
||||
|
||||
# Neighbouring steps are the value a row reads when the broadcast stride is
|
||||
# ignored; the production tensor reads uninitialized memory instead.
|
||||
neighbours = torch.tensor([2, 0, 0, 0]).cuda()
|
||||
broadcast, broadcast_prior = run(
|
||||
neighbours[0].expand(num_rows), torch.device("cuda")
|
||||
)
|
||||
assert torch.equal(broadcast, expected)
|
||||
assert torch.equal(broadcast_prior, expected_prior)
|
||||
|
||||
step = torch.tensor(2).cuda()
|
||||
production, production_prior = run(step.expand(num_rows), torch.device("cuda"))
|
||||
assert torch.equal(production, expected)
|
||||
assert torch.equal(production_prior, expected_prior)
|
||||
|
||||
|
||||
def test_draft_sampler_uses_draft_key_and_advances_context(monkeypatch):
|
||||
class StubSpeculator(DraftModelSpeculator):
|
||||
def capture(self): ...
|
||||
@@ -937,22 +1188,32 @@ def test_draft_sampler_uses_draft_key_and_advances_context(monkeypatch):
|
||||
return torch.zeros(hidden_states.shape[0], 8)
|
||||
|
||||
class StubWatermarker:
|
||||
context_width = 2
|
||||
|
||||
@staticmethod
|
||||
def sample(logits, contexts, random_sampler=None):
|
||||
assert random_sampler is None
|
||||
return WatermarkSample(torch.tensor([7, 7]), logits)
|
||||
def sample(logits, contexts, random_sampler=None, skip_mask=None):
|
||||
assert random_sampler is not None
|
||||
return WatermarkSample(
|
||||
torch.where(skip_mask, random_sampler(logits), 7), logits
|
||||
)
|
||||
|
||||
speculator = object.__new__(StubSpeculator)
|
||||
speculator.model = StubModel()
|
||||
speculator.use_fp64_gumbel = False
|
||||
speculator.acceptance_estimator = None
|
||||
draft_watermarker = object.__new__(DraftWatermarker)
|
||||
draft_watermarker.watermarker = StubWatermarker()
|
||||
draft_watermarker.contexts = torch.tensor([[1, 2], [3, 4]])
|
||||
draft_watermarker = DraftWatermarker(
|
||||
StubWatermarker(),
|
||||
max_num_reqs=2,
|
||||
device=torch.device("cpu"),
|
||||
num_speculative_steps=1,
|
||||
deduplicate_contexts="none",
|
||||
deduplicate_contexts_max_history=None,
|
||||
)
|
||||
draft_watermarker.contexts.copy_(torch.tensor([[1, 2], [3, 4]]))
|
||||
draft_watermarker.enabled = torch.tensor([True, False])
|
||||
speculator.draft_watermarker = draft_watermarker
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.worker.gpu.spec_decode.speculator.gumbel_sample",
|
||||
"vllm.v1.watermarking.watermarker.gumbel_sample",
|
||||
lambda *args, **kwargs: torch.tensor([3, 4]),
|
||||
)
|
||||
|
||||
@@ -970,6 +1231,274 @@ def test_draft_sampler_uses_draft_key_and_advances_context(monkeypatch):
|
||||
assert torch.equal(draft_watermarker.contexts, torch.tensor([[2, 7], [4, 4]]))
|
||||
|
||||
|
||||
def test_draft_sampler_uses_ordinary_samples_for_repeated_contexts(monkeypatch):
|
||||
class StubWatermarker:
|
||||
context_width = 1
|
||||
|
||||
def __init__(self):
|
||||
self.samples = iter((2, 1, 8))
|
||||
|
||||
def sample(self, logits, contexts, random_sampler=None, skip_mask=None):
|
||||
watermarked = torch.tensor([next(self.samples)])
|
||||
return WatermarkSample(
|
||||
torch.where(skip_mask, random_sampler(logits), watermarked), logits
|
||||
)
|
||||
|
||||
draft_watermarker = DraftWatermarker(
|
||||
StubWatermarker(),
|
||||
max_num_reqs=1,
|
||||
device=torch.device("cpu"),
|
||||
num_speculative_steps=3,
|
||||
deduplicate_contexts="single_turn",
|
||||
deduplicate_contexts_max_history=None,
|
||||
)
|
||||
draft_watermarker.prepare(
|
||||
contexts=torch.tensor([[1]]),
|
||||
enabled=torch.tensor([True]),
|
||||
all_token_ids=torch.tensor([[100, 1, 0, 0]]),
|
||||
prompt_lens=torch.tensor([1]),
|
||||
total_lens=torch.tensor([2]),
|
||||
)
|
||||
idx_mapping = torch.tensor([0])
|
||||
temperature = torch.tensor([1.0])
|
||||
ordinary_samples = iter((torch.tensor([5]), torch.tensor([6]), torch.tensor([4])))
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.watermarking.watermarker.gumbel_sample",
|
||||
lambda *args, **kwargs: next(ordinary_samples),
|
||||
)
|
||||
seeds = torch.tensor([0])
|
||||
draft_logits = torch.zeros(1, 3, 8)
|
||||
|
||||
first = draft_watermarker.sample(
|
||||
torch.zeros(1, 8),
|
||||
idx_mapping,
|
||||
temperature=temperature,
|
||||
seed=seeds,
|
||||
pos=torch.tensor([0]),
|
||||
apply_temperature=True,
|
||||
is_drafting=True,
|
||||
logits_cache_col=torch.tensor(0),
|
||||
logits_cache=draft_logits,
|
||||
use_fp64=False,
|
||||
)
|
||||
second = draft_watermarker.sample(
|
||||
torch.zeros(1, 8),
|
||||
idx_mapping,
|
||||
temperature=temperature,
|
||||
seed=seeds,
|
||||
pos=torch.tensor([1]),
|
||||
apply_temperature=True,
|
||||
is_drafting=True,
|
||||
logits_cache_col=torch.tensor(1),
|
||||
logits_cache=draft_logits,
|
||||
use_fp64=False,
|
||||
)
|
||||
repeated = draft_watermarker.sample(
|
||||
torch.zeros(1, 8),
|
||||
idx_mapping,
|
||||
temperature=temperature,
|
||||
seed=seeds,
|
||||
pos=torch.tensor([2]),
|
||||
apply_temperature=True,
|
||||
is_drafting=True,
|
||||
logits_cache_col=torch.tensor(2),
|
||||
logits_cache=draft_logits,
|
||||
use_fp64=False,
|
||||
)
|
||||
|
||||
assert first.item() == 2
|
||||
assert second.item() == 1
|
||||
assert repeated.item() == 4
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
|
||||
)
|
||||
def test_draft_context_deduplication_reads_strided_request_indices():
|
||||
"""DSpark samples a step at a time, so req_indices is an `idx_map[:, i]` column."""
|
||||
num_reqs = 4
|
||||
num_speculative_steps = 3
|
||||
all_token_ids = torch.tensor(
|
||||
[[3, 4, 3, 4], [9, 9, 9, 9], [3, 4, 3, 4], [9, 9, 9, 9]]
|
||||
).cuda()
|
||||
prompt_lens = torch.zeros(num_reqs, dtype=torch.int64).cuda()
|
||||
total_lens = torch.full((num_reqs,), 4).cuda()
|
||||
contexts = torch.tensor([[3, 4]]).expand(num_reqs, 2).contiguous().cuda()
|
||||
steps = torch.zeros(num_reqs, dtype=torch.int64).cuda()
|
||||
enabled = torch.ones(num_reqs, dtype=torch.bool).cuda()
|
||||
|
||||
idx_map = (
|
||||
torch.arange(num_reqs)
|
||||
.repeat_interleave(num_speculative_steps)
|
||||
.view(num_reqs, num_speculative_steps)
|
||||
.cuda()
|
||||
)
|
||||
column = idx_map[:, 1]
|
||||
assert column.stride(0) == num_speculative_steps
|
||||
|
||||
def run(req_indices):
|
||||
prior = torch.zeros(
|
||||
num_reqs, num_speculative_steps, 2, dtype=torch.int64, device="cuda"
|
||||
)
|
||||
return draft_watermarking_mask(
|
||||
all_token_ids,
|
||||
req_indices,
|
||||
prompt_lens,
|
||||
total_lens,
|
||||
prior,
|
||||
contexts,
|
||||
steps,
|
||||
enabled,
|
||||
max_history=None,
|
||||
include_prompt=False,
|
||||
).cpu()
|
||||
|
||||
expected = torch.tensor([False, True, False, True])
|
||||
assert torch.equal(run(column.contiguous()), expected)
|
||||
assert torch.equal(run(column), expected)
|
||||
|
||||
|
||||
def test_draft_sampler_deduplicates_against_committed_history(monkeypatch):
|
||||
class StubWatermarker:
|
||||
context_width = 1
|
||||
|
||||
@staticmethod
|
||||
def sample(logits, contexts, random_sampler=None, skip_mask=None):
|
||||
return WatermarkSample(
|
||||
torch.where(skip_mask, random_sampler(logits), 7), logits
|
||||
)
|
||||
|
||||
draft_watermarker = DraftWatermarker(
|
||||
StubWatermarker(),
|
||||
max_num_reqs=1,
|
||||
device=torch.device("cpu"),
|
||||
num_speculative_steps=1,
|
||||
deduplicate_contexts="single_turn",
|
||||
deduplicate_contexts_max_history=None,
|
||||
)
|
||||
draft_watermarker.prepare(
|
||||
contexts=torch.tensor([[1]]),
|
||||
enabled=torch.tensor([True]),
|
||||
all_token_ids=torch.tensor([[100, 1, 2, 1]]),
|
||||
prompt_lens=torch.tensor([1]),
|
||||
total_lens=torch.tensor([4]),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.watermarking.watermarker.gumbel_sample",
|
||||
lambda *args, **kwargs: torch.tensor([4]),
|
||||
)
|
||||
sampled = draft_watermarker.sample(
|
||||
torch.zeros(1, 8),
|
||||
torch.tensor([0]),
|
||||
temperature=torch.tensor([1.0]),
|
||||
seed=torch.tensor([0]),
|
||||
pos=torch.tensor([0]),
|
||||
apply_temperature=True,
|
||||
is_drafting=True,
|
||||
logits_cache_col=torch.tensor(0),
|
||||
logits_cache=torch.zeros(1, 1, 8),
|
||||
use_fp64=False,
|
||||
)
|
||||
|
||||
assert sampled.item() == 4
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
|
||||
)
|
||||
@pytest.mark.parametrize("idx_mapping_layout", ["contiguous", "column_view"])
|
||||
def test_draft_watermarker_deduplicates_across_a_block_on_cuda(
|
||||
monkeypatch, idx_mapping_layout
|
||||
):
|
||||
"""Drive the whole draft sampler over a block with the production shapes.
|
||||
|
||||
`draft_step` is a 0-d tensor, and a speculator that samples one step at a
|
||||
time hands over a column of its index map. Each step is checked against the
|
||||
committed history the window still reaches and against the earlier steps of
|
||||
the block.
|
||||
"""
|
||||
num_reqs, num_steps, vocab_size = 2, 3, 16
|
||||
watermarker = GumbelWatermarker(key=42, context_width=2)
|
||||
draft_watermarker = DraftWatermarker(
|
||||
watermarker,
|
||||
max_num_reqs=num_reqs,
|
||||
device=torch.device("cuda"),
|
||||
num_speculative_steps=num_steps,
|
||||
deduplicate_contexts="single_turn",
|
||||
deduplicate_contexts_max_history=4,
|
||||
)
|
||||
draft_watermarker.prepare(
|
||||
contexts=torch.tensor([[5, 6], [6, 3]]).cuda(),
|
||||
enabled=torch.ones(num_reqs, dtype=torch.bool).cuda(),
|
||||
all_token_ids=torch.tensor([[5, 6, 7, 5, 6], [8, 9, 6, 6, 3]]).cuda(),
|
||||
prompt_lens=torch.zeros(num_reqs, dtype=torch.int64).cuda(),
|
||||
total_lens=torch.full((num_reqs,), 5).cuda(),
|
||||
)
|
||||
|
||||
if idx_mapping_layout == "contiguous":
|
||||
idx_mapping = torch.arange(num_reqs, dtype=torch.int32).cuda()
|
||||
else:
|
||||
idx_mapping = (
|
||||
torch.arange(num_reqs, dtype=torch.int32)
|
||||
.repeat_interleave(num_steps)
|
||||
.view(num_reqs, num_steps)
|
||||
.cuda()
|
||||
)[:, 1]
|
||||
assert idx_mapping.stride(0) == num_steps
|
||||
|
||||
skip_masks: list[torch.Tensor] = []
|
||||
original = philox_gumbel_sample
|
||||
|
||||
def spy(*args, **kwargs):
|
||||
skip_masks.append(kwargs["skip_mask"].cpu())
|
||||
return original(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr("vllm.v1.worker.gpu.sample.watermark.philox_gumbel_sample", spy)
|
||||
|
||||
draft_logits = torch.zeros(num_reqs, num_steps, vocab_size + 1).cuda()
|
||||
step_logits = []
|
||||
for step in range(num_steps):
|
||||
# Token 6 wins by a margin no Gumbel draw can close, on either path.
|
||||
logits = torch.zeros(num_reqs, vocab_size).cuda()
|
||||
logits[:, 6] = 100.0
|
||||
logits[:, 0] = float(step) + 1.0
|
||||
step_logits.append(logits)
|
||||
sampled = draft_watermarker.sample(
|
||||
logits,
|
||||
idx_mapping,
|
||||
temperature=torch.ones(num_reqs).cuda(),
|
||||
seed=torch.zeros(num_reqs, dtype=torch.int64).cuda(),
|
||||
pos=torch.full((num_reqs,), 5 + step, dtype=torch.int64).cuda(),
|
||||
apply_temperature=True,
|
||||
is_drafting=True,
|
||||
logits_cache_col=torch.tensor(step).cuda(),
|
||||
logits_cache=draft_logits,
|
||||
use_fp64=False,
|
||||
)
|
||||
assert torch.equal(sampled.cpu(), torch.tensor([6, 6]))
|
||||
|
||||
# Step 0: request 0's [5, 6] is committed at position 2, request 1's [6, 3]
|
||||
# is new. Step 1: [6, 6] and [3, 6] are both new. Step 2: request 0's
|
||||
# [6, 6] repeats its own step-1 context and request 1's repeats position 4.
|
||||
assert [mask.tolist() for mask in skip_masks] == [
|
||||
[True, False],
|
||||
[False, False],
|
||||
[True, True],
|
||||
]
|
||||
assert torch.equal(
|
||||
draft_watermarker.prior_contexts[:num_reqs].cpu(),
|
||||
torch.tensor([[[5, 6], [6, 6], [6, 6]], [[6, 3], [3, 6], [6, 6]]]),
|
||||
)
|
||||
assert torch.equal(
|
||||
draft_watermarker.contexts[:num_reqs].cpu(), torch.tensor([[6, 6], [6, 6]])
|
||||
)
|
||||
for step, logits in enumerate(step_logits):
|
||||
# The 0-d draft step selects the logits cache column.
|
||||
assert torch.equal(draft_logits[:, step, :vocab_size], logits)
|
||||
assert not draft_logits[:, :, -1].any()
|
||||
|
||||
|
||||
def test_dspark_reduced_vocab_draft_sampler_applies_watermarking(monkeypatch):
|
||||
speculator = object.__new__(DSparkSpeculator)
|
||||
speculator.draft_logits = torch.zeros(2, 1, 8)
|
||||
@@ -982,15 +1511,23 @@ def test_dspark_reduced_vocab_draft_sampler_applies_watermarking(monkeypatch):
|
||||
speculator.acceptance_estimator = None
|
||||
watermark_logits: list[torch.Tensor] = []
|
||||
|
||||
def sample(logits, sampled, idx_map, temperature):
|
||||
def sample(
|
||||
logits,
|
||||
idx_mapping,
|
||||
temperature,
|
||||
seed,
|
||||
pos,
|
||||
apply_temperature,
|
||||
is_drafting,
|
||||
logits_cache=None,
|
||||
logits_cache_col=None,
|
||||
use_fp64=False,
|
||||
logits_cache_source=None,
|
||||
):
|
||||
watermark_logits.append(logits.clone())
|
||||
return sampled + 1
|
||||
return torch.tensor([4, 5])
|
||||
|
||||
speculator.draft_watermarker = SimpleNamespace(sample=sample)
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.worker.gpu.spec_decode.dspark.speculator.gumbel_sample",
|
||||
lambda *args, **kwargs: torch.tensor([3, 4]),
|
||||
)
|
||||
|
||||
sampled = speculator._sample_logits(
|
||||
torch.tensor([[10.0, 20.0], [30.0, 40.0]]),
|
||||
|
||||
@@ -1344,13 +1344,6 @@ class VllmConfig:
|
||||
"share of output tokens supplied by accepted drafts.",
|
||||
scope="global",
|
||||
)
|
||||
if watermark_config.deduplicate_contexts != "none":
|
||||
logger.warning_once(
|
||||
"Context deduplication is not supported with speculative "
|
||||
"decoding and will not be applied to accepted drafts, "
|
||||
"rejection-recovery tokens, or bonus tokens.",
|
||||
scope="global",
|
||||
)
|
||||
if watermark_config.algorithm == "dual_key_gumbel" and (
|
||||
watermark_config.alpha != get_field(WatermarkConfig, "alpha").default
|
||||
):
|
||||
|
||||
@@ -119,8 +119,9 @@ class GPUWatermarkSampler(Sampler):
|
||||
self,
|
||||
expanded_idx_mapping: torch.Tensor,
|
||||
contexts: torch.Tensor,
|
||||
expanded_local_pos: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return repeated_context_mask(
|
||||
repeated = repeated_context_mask(
|
||||
self.req_states.all_token_ids.gpu,
|
||||
expanded_idx_mapping,
|
||||
self.req_states.prompt_len.gpu,
|
||||
@@ -128,8 +129,16 @@ class GPUWatermarkSampler(Sampler):
|
||||
contexts,
|
||||
self.deduplicate_contexts_max_history,
|
||||
include_prompt=self.deduplicate_contexts == "all",
|
||||
skip_partial_context=self.deduplicate_contexts == "all",
|
||||
skip_partial_context=(
|
||||
self.deduplicate_contexts == "all" and expanded_local_pos is None
|
||||
),
|
||||
history_offsets=expanded_local_pos,
|
||||
local_positions=expanded_local_pos,
|
||||
num_speculative_steps=(
|
||||
self.num_speculative_tokens if expanded_local_pos is not None else 0
|
||||
),
|
||||
)
|
||||
return repeated
|
||||
|
||||
def _get_contexts(
|
||||
self,
|
||||
|
||||
@@ -94,6 +94,10 @@ class GumbelWatermarker(Watermarker):
|
||||
seeds=random_sampler.seeds,
|
||||
positions=random_sampler.positions,
|
||||
use_fp64=random_sampler.use_fp64,
|
||||
is_drafting=random_sampler.is_drafting,
|
||||
logits_cache=random_sampler.logits_cache,
|
||||
logits_cache_col=random_sampler.logits_cache_col,
|
||||
logits_cache_source=random_sampler.logits_cache_source,
|
||||
)
|
||||
return WatermarkSample(token_ids, logits)
|
||||
|
||||
|
||||
@@ -3,14 +3,16 @@
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config.watermarking import WatermarkConfig
|
||||
from vllm.config.watermarking import WatermarkConfig, WatermarkContextScope
|
||||
from vllm.v1.watermarking.factory import create_watermarker
|
||||
from vllm.v1.watermarking.gumbel import GumbelWatermarker
|
||||
from vllm.v1.watermarking.prfs import PhiloxPRF
|
||||
from vllm.v1.watermarking.watermarker import (
|
||||
RandomSampler,
|
||||
SupportsSpeculativeDecoding,
|
||||
Watermarker,
|
||||
)
|
||||
from vllm.v1.worker.gpu.sample.watermark import draft_watermarking_mask
|
||||
from vllm.v1.worker.gpu.spec_decode.rejection_sampler_utils import rejection_sample
|
||||
|
||||
|
||||
@@ -20,8 +22,14 @@ class DraftWatermarker:
|
||||
watermarker: Watermarker,
|
||||
max_num_reqs: int,
|
||||
device: torch.device,
|
||||
num_speculative_steps: int,
|
||||
deduplicate_contexts: WatermarkContextScope,
|
||||
deduplicate_contexts_max_history: int | None,
|
||||
) -> None:
|
||||
self.watermarker = watermarker
|
||||
self.num_speculative_steps = num_speculative_steps
|
||||
self.deduplicate_contexts = deduplicate_contexts
|
||||
self.deduplicate_contexts_max_history = deduplicate_contexts_max_history
|
||||
self.contexts = torch.zeros(
|
||||
max_num_reqs,
|
||||
watermarker.context_width,
|
||||
@@ -29,30 +37,112 @@ class DraftWatermarker:
|
||||
device=device,
|
||||
)
|
||||
self.enabled = torch.zeros(max_num_reqs, dtype=torch.bool, device=device)
|
||||
self.prior_contexts = torch.zeros(
|
||||
max_num_reqs,
|
||||
num_speculative_steps,
|
||||
watermarker.context_width,
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
)
|
||||
self.all_token_ids: torch.Tensor | None = None
|
||||
self.prompt_lens: torch.Tensor | None = None
|
||||
self.total_lens: torch.Tensor | None = None
|
||||
|
||||
def prepare(self, contexts: torch.Tensor, enabled: torch.Tensor) -> None:
|
||||
def prepare(
|
||||
self,
|
||||
contexts: torch.Tensor,
|
||||
enabled: torch.Tensor,
|
||||
all_token_ids: torch.Tensor,
|
||||
prompt_lens: torch.Tensor,
|
||||
total_lens: torch.Tensor,
|
||||
) -> None:
|
||||
num_reqs = contexts.shape[0]
|
||||
self.contexts[:num_reqs].copy_(contexts)
|
||||
self.enabled[:num_reqs].copy_(enabled)
|
||||
self.all_token_ids = all_token_ids
|
||||
self.prompt_lens = prompt_lens
|
||||
self.total_lens = total_lens
|
||||
|
||||
def _sampling_state(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
idx_mapping: torch.Tensor,
|
||||
request_temperatures: torch.Tensor,
|
||||
draft_step: int | torch.Tensor,
|
||||
contexts: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
num_rows = logits.shape[0]
|
||||
steps = torch.as_tensor(draft_step, device=logits.device, dtype=torch.int64)
|
||||
if steps.ndim == 0:
|
||||
steps = steps.expand(num_rows)
|
||||
enabled = self.enabled[:num_rows] & (request_temperatures != 0)
|
||||
if self.deduplicate_contexts == "none":
|
||||
return steps, enabled
|
||||
assert self.all_token_ids is not None
|
||||
assert self.prompt_lens is not None
|
||||
assert self.total_lens is not None
|
||||
enabled = draft_watermarking_mask(
|
||||
self.all_token_ids,
|
||||
idx_mapping,
|
||||
self.prompt_lens,
|
||||
self.total_lens,
|
||||
self.prior_contexts,
|
||||
contexts,
|
||||
steps,
|
||||
enabled,
|
||||
self.deduplicate_contexts_max_history,
|
||||
include_prompt=self.deduplicate_contexts == "all",
|
||||
)
|
||||
return steps, enabled
|
||||
|
||||
def sample(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
ordinary_sampled: torch.Tensor,
|
||||
idx_mapping: torch.Tensor,
|
||||
temperature: torch.Tensor,
|
||||
seed: torch.Tensor,
|
||||
pos: torch.Tensor,
|
||||
apply_temperature: bool,
|
||||
is_drafting: bool,
|
||||
logits_cache: torch.Tensor | None = None,
|
||||
logits_cache_col: torch.Tensor | None = None,
|
||||
use_fp64: bool = False,
|
||||
logits_cache_source: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert apply_temperature
|
||||
assert is_drafting
|
||||
assert logits_cache is not None
|
||||
assert logits_cache_col is not None
|
||||
num_rows = logits.shape[0]
|
||||
request_temperatures = temperature[idx_mapping]
|
||||
contexts = self.contexts[:num_rows]
|
||||
steps, enabled = self._sampling_state(
|
||||
logits, idx_mapping, request_temperatures, logits_cache_col, contexts
|
||||
)
|
||||
|
||||
processed_logits = logits / torch.where(
|
||||
request_temperatures == 0, 1, request_temperatures
|
||||
).unsqueeze(-1)
|
||||
watermarked = self.watermarker.sample(
|
||||
random_sampler = RandomSampler(
|
||||
expanded_idx_mapping=idx_mapping,
|
||||
temperatures=temperature,
|
||||
seeds=seed,
|
||||
positions=pos,
|
||||
use_fp64=use_fp64,
|
||||
is_drafting=True,
|
||||
logits_cache=logits_cache,
|
||||
logits_cache_col=steps,
|
||||
logits_cache_source=(
|
||||
logits if logits_cache_source is None else logits_cache_source
|
||||
),
|
||||
)
|
||||
sampled = self.watermarker.sample(
|
||||
processed_logits,
|
||||
self.contexts[: logits.shape[0]],
|
||||
contexts,
|
||||
random_sampler=random_sampler,
|
||||
skip_mask=~enabled,
|
||||
).token_ids
|
||||
enabled = self.enabled[: logits.shape[0]] & (request_temperatures != 0)
|
||||
sampled = torch.where(enabled, watermarked, ordinary_sampled)
|
||||
contexts = self.contexts[: logits.shape[0]]
|
||||
|
||||
contexts.copy_(torch.cat((contexts[:, 1:], sampled.unsqueeze(-1)), dim=-1))
|
||||
return sampled
|
||||
|
||||
@@ -68,9 +158,19 @@ def create_speculative_draft_watermarker(
|
||||
max_num_reqs: int,
|
||||
device: torch.device,
|
||||
allow_target_only: bool,
|
||||
num_speculative_steps: int = 1,
|
||||
deduplicate_contexts: WatermarkContextScope = "none",
|
||||
deduplicate_contexts_max_history: int | None = 8192,
|
||||
) -> DraftWatermarker | None:
|
||||
if isinstance(watermarker, SupportsSpeculativeDecoding):
|
||||
return DraftWatermarker(watermarker.draft_watermarker, max_num_reqs, device)
|
||||
return DraftWatermarker(
|
||||
watermarker.draft_watermarker,
|
||||
max_num_reqs,
|
||||
device,
|
||||
num_speculative_steps,
|
||||
deduplicate_contexts,
|
||||
deduplicate_contexts_max_history,
|
||||
)
|
||||
if allow_target_only:
|
||||
return None
|
||||
raise ValueError(
|
||||
@@ -130,9 +230,10 @@ def watermarked_rejection_sample(
|
||||
contexts: torch.Tensor,
|
||||
watermarking: torch.Tensor,
|
||||
watermarker: Watermarker,
|
||||
watermarking_skip_mask: torch.Tensor | None = None,
|
||||
use_fp64: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Use the target key for rejection recovery and bonus tokens."""
|
||||
"""Use the target key except at contexts selected for ordinary sampling."""
|
||||
assert contexts.shape == (target_logits.shape[0], watermarker.context_width)
|
||||
return rejection_sample(
|
||||
target_logits,
|
||||
@@ -149,5 +250,6 @@ def watermarked_rejection_sample(
|
||||
use_fp64=use_fp64,
|
||||
contexts=contexts,
|
||||
watermarking=watermarking,
|
||||
watermarking_skip_mask=watermarking_skip_mask,
|
||||
watermark_key=_resolve_watermark_key(watermarker),
|
||||
)
|
||||
|
||||
@@ -23,6 +23,10 @@ class RandomSampler:
|
||||
seeds: torch.Tensor
|
||||
positions: torch.Tensor
|
||||
use_fp64: bool = False
|
||||
is_drafting: bool = False
|
||||
logits_cache: torch.Tensor | None = None
|
||||
logits_cache_col: torch.Tensor | None = None
|
||||
logits_cache_source: torch.Tensor | None = None
|
||||
|
||||
def __call__(self, logits: torch.Tensor) -> torch.Tensor:
|
||||
return gumbel_sample(
|
||||
@@ -32,7 +36,10 @@ class RandomSampler:
|
||||
self.seeds,
|
||||
self.positions,
|
||||
apply_temperature=False,
|
||||
is_drafting=False,
|
||||
is_drafting=self.is_drafting,
|
||||
logits_cache=self.logits_cache,
|
||||
logits_cache_col=self.logits_cache_col,
|
||||
logits_cache_source=self.logits_cache_source,
|
||||
use_fp64=self.use_fp64,
|
||||
)
|
||||
|
||||
|
||||
@@ -914,8 +914,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
spec_hidden_states = pre_hc_hidden_states[: hidden_states.shape[0]] # type: ignore[union-attr]
|
||||
if isinstance(self.sampler, GPUWatermarkSampler):
|
||||
self.speculator.prepare_watermarking(
|
||||
self.sampler._get_contexts(input_batch.idx_mapping),
|
||||
self.sampler.watermarking.gpu[input_batch.idx_mapping],
|
||||
self.sampler, input_batch.idx_mapping
|
||||
)
|
||||
with use_workspace_lane(self._draft_workspace_lane):
|
||||
self.speculator.propose(
|
||||
@@ -2194,8 +2193,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
spec_hidden_states = pre_hc_hidden_states[: draft_hidden_states.size(0)]
|
||||
if isinstance(self.sampler, GPUWatermarkSampler):
|
||||
self.speculator.prepare_watermarking(
|
||||
self.sampler._get_contexts(input_batch.idx_mapping),
|
||||
self.sampler.watermarking.gpu[input_batch.idx_mapping],
|
||||
self.sampler, input_batch.idx_mapping
|
||||
)
|
||||
with use_workspace_lane(self._draft_workspace_lane):
|
||||
draft_tokens = self.speculator.propose(
|
||||
|
||||
@@ -220,6 +220,8 @@ def gumbel_block_argmax(
|
||||
logits_cache_stride_0,
|
||||
logits_cache_stride_1,
|
||||
logits_cache_col_ptr,
|
||||
logits_cache_source_ptr,
|
||||
logits_cache_source_stride,
|
||||
vocab_size,
|
||||
IS_DRAFTING: tl.constexpr,
|
||||
APPLY_TEMPERATURE: tl.constexpr,
|
||||
@@ -240,12 +242,16 @@ def gumbel_block_argmax(
|
||||
col = tl.load(logits_cache_col_ptr + token_idx)
|
||||
else:
|
||||
col = tl.load(logits_cache_col_ptr)
|
||||
cached_logits = tl.load(
|
||||
logits_cache_source_ptr + token_idx * logits_cache_source_stride + block,
|
||||
mask=mask,
|
||||
)
|
||||
tl.store(
|
||||
logits_cache_ptr
|
||||
+ req_state_idx * logits_cache_stride_0
|
||||
+ col * logits_cache_stride_1
|
||||
+ block,
|
||||
logits,
|
||||
cached_logits,
|
||||
mask=mask & is_valid_req,
|
||||
)
|
||||
|
||||
@@ -275,6 +281,8 @@ def _gumbel_sample_kernel(
|
||||
logits_cache_stride_0,
|
||||
logits_cache_stride_1,
|
||||
logits_cache_col_ptr,
|
||||
logits_cache_source_ptr,
|
||||
logits_cache_source_stride,
|
||||
logits_ptr,
|
||||
logits_stride,
|
||||
expanded_idx_mapping_ptr,
|
||||
@@ -312,6 +320,8 @@ def _gumbel_sample_kernel(
|
||||
logits_cache_stride_0,
|
||||
logits_cache_stride_1,
|
||||
logits_cache_col_ptr,
|
||||
logits_cache_source_ptr,
|
||||
logits_cache_source_stride,
|
||||
vocab_size,
|
||||
IS_DRAFTING=IS_DRAFTING,
|
||||
APPLY_TEMPERATURE=APPLY_TEMPERATURE,
|
||||
@@ -334,6 +344,7 @@ def gumbel_sample(
|
||||
logits_cache: torch.Tensor | None = None, # [max_num_reqs, num_cols, vocab_size]
|
||||
logits_cache_col: torch.Tensor | None = None, # scalar or [num_tokens]
|
||||
use_fp64: bool = False,
|
||||
logits_cache_source: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
# Enforce contiguity on non-strided input tensors
|
||||
expanded_idx_mapping = expanded_idx_mapping.contiguous()
|
||||
@@ -342,11 +353,26 @@ def gumbel_sample(
|
||||
logits_cache_col = logits_cache_col.contiguous()
|
||||
num_tokens, vocab_size = logits.shape
|
||||
if logits_cache is not None:
|
||||
if logits_cache_source is None:
|
||||
logits_cache_source = logits
|
||||
assert logits_cache_source.shape == logits.shape, (
|
||||
"logits cache source must match sampled logits shape"
|
||||
)
|
||||
assert logits_cache_source.device == logits.device, (
|
||||
"logits cache source must be on the sampled logits device"
|
||||
)
|
||||
assert logits_cache_source.dtype == logits_cache.dtype, (
|
||||
"logits cache source and destination must have the same dtype"
|
||||
)
|
||||
assert logits_cache.size(-1) >= vocab_size, (
|
||||
f"draft logits cache vocab dim ({logits_cache.size(-1)}) is narrower "
|
||||
f"than the sampled logits ({vocab_size}). Cached logits would be "
|
||||
"truncated."
|
||||
)
|
||||
elif logits_cache_source is not None:
|
||||
raise ValueError("logits_cache_source requires logits_cache")
|
||||
if logits_cache_source is not None and logits_cache_source.stride(-1) != 1:
|
||||
logits_cache_source = logits_cache_source.contiguous()
|
||||
BLOCK_SIZE = 1024
|
||||
num_blocks = triton.cdiv(vocab_size, BLOCK_SIZE)
|
||||
local_argmax = logits.new_empty(num_tokens, num_blocks, dtype=torch.int64)
|
||||
@@ -362,6 +388,8 @@ def gumbel_sample(
|
||||
logits_cache.stride(0) if logits_cache is not None else 0,
|
||||
logits_cache.stride(1) if logits_cache is not None else 0,
|
||||
logits_cache_col,
|
||||
logits_cache_source,
|
||||
logits_cache_source.stride(0) if logits_cache_source is not None else 0,
|
||||
logits,
|
||||
logits.stride(0),
|
||||
expanded_idx_mapping,
|
||||
|
||||
@@ -51,12 +51,25 @@ def _repeated_context_mask_kernel(
|
||||
req_indices_ptr,
|
||||
prompt_lens_ptr,
|
||||
total_lens_ptr,
|
||||
history_offsets_ptr,
|
||||
history_offsets_stride,
|
||||
contexts_ptr,
|
||||
context_stride,
|
||||
local_positions_ptr,
|
||||
prior_contexts_ptr,
|
||||
prior_contexts_stride_0,
|
||||
prior_contexts_stride_1,
|
||||
steps_ptr,
|
||||
steps_stride,
|
||||
enabled_ptr,
|
||||
CONTEXT_WIDTH: tl.constexpr,
|
||||
NUM_SPECULATIVE_STEPS: tl.constexpr,
|
||||
MAX_HISTORY: tl.constexpr,
|
||||
INCLUDE_PROMPT: tl.constexpr,
|
||||
SKIP_PARTIAL_CONTEXT: tl.constexpr,
|
||||
HAS_HISTORY_OFFSETS: tl.constexpr,
|
||||
SPECULATIVE_CONTEXTS: tl.constexpr,
|
||||
DRAFT_CONTEXTS: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
@@ -71,12 +84,16 @@ def _repeated_context_mask_kernel(
|
||||
return
|
||||
sequence_start = 0 if INCLUDE_PROMPT else prompt_len
|
||||
history_len = total_len - sequence_start
|
||||
history_offset = 0
|
||||
if HAS_HISTORY_OFFSETS:
|
||||
history_offset = tl.load(history_offsets_ptr + row * history_offsets_stride)
|
||||
|
||||
offsets = tl.arange(0, BLOCK)
|
||||
repeated = tl.full((), 0, tl.int32)
|
||||
scan_start = 0
|
||||
if MAX_HISTORY > 0:
|
||||
scan_start = tl.maximum(history_len - MAX_HISTORY, 0)
|
||||
remaining_history = tl.maximum(MAX_HISTORY - history_offset, 0)
|
||||
scan_start = tl.maximum(history_len - remaining_history, 0)
|
||||
# Aligned loop bounds let the per-lane offsets fold into immediates.
|
||||
aligned_start = (scan_start // BLOCK) * BLOCK
|
||||
for block_start in tl.range(aligned_start, history_len, BLOCK):
|
||||
@@ -96,6 +113,93 @@ def _repeated_context_mask_kernel(
|
||||
)
|
||||
matches &= historical_token == context_token
|
||||
repeated |= tl.max(matches.to(tl.int32), axis=0)
|
||||
|
||||
if SPECULATIVE_CONTEXTS:
|
||||
local_pos = tl.load(local_positions_ptr + row)
|
||||
if INCLUDE_PROMPT:
|
||||
partial_context = tl.full((), 0, tl.int32)
|
||||
for context_offset in range(CONTEXT_WIDTH):
|
||||
context_token = tl.load(
|
||||
contexts_ptr + row * context_stride + context_offset
|
||||
)
|
||||
partial_context |= context_token < 0
|
||||
repeated |= partial_context
|
||||
for offset in range(1, NUM_SPECULATIVE_STEPS + 1):
|
||||
prior_row = row - offset
|
||||
in_history = (prior_row >= 0) & (local_pos >= offset)
|
||||
if MAX_HISTORY > 0:
|
||||
in_history &= offset <= MAX_HISTORY
|
||||
prior_req_idx = tl.load(
|
||||
req_indices_ptr + tl.maximum(prior_row, 0),
|
||||
mask=in_history,
|
||||
other=-1,
|
||||
)
|
||||
prior_local_pos = tl.load(
|
||||
local_positions_ptr + tl.maximum(prior_row, 0),
|
||||
mask=in_history,
|
||||
other=-1,
|
||||
)
|
||||
matches = (
|
||||
in_history
|
||||
& (req_idx == prior_req_idx)
|
||||
& (local_pos == prior_local_pos + offset)
|
||||
)
|
||||
for context_offset in range(CONTEXT_WIDTH):
|
||||
context_token = tl.load(
|
||||
contexts_ptr + row * context_stride + context_offset
|
||||
)
|
||||
prior_token = tl.load(
|
||||
contexts_ptr
|
||||
+ tl.maximum(prior_row, 0) * context_stride
|
||||
+ context_offset,
|
||||
mask=in_history,
|
||||
other=-2,
|
||||
)
|
||||
matches &= context_token == prior_token
|
||||
repeated |= matches
|
||||
|
||||
if DRAFT_CONTEXTS:
|
||||
step = tl.load(steps_ptr + row * steps_stride).to(tl.int64)
|
||||
partial_context = tl.full((), 0, tl.int32)
|
||||
for context_offset in range(CONTEXT_WIDTH):
|
||||
context_token = tl.load(
|
||||
contexts_ptr + row * context_stride + context_offset
|
||||
)
|
||||
partial_context |= context_token < 0
|
||||
if INCLUDE_PROMPT:
|
||||
repeated |= partial_context
|
||||
|
||||
for prior_step in range(NUM_SPECULATIVE_STEPS):
|
||||
in_history = prior_step < step
|
||||
if MAX_HISTORY > 0:
|
||||
in_history &= prior_step >= step - MAX_HISTORY
|
||||
matches = in_history
|
||||
for context_offset in range(CONTEXT_WIDTH):
|
||||
context_token = tl.load(
|
||||
contexts_ptr + row * context_stride + context_offset
|
||||
)
|
||||
prior_token = tl.load(
|
||||
prior_contexts_ptr
|
||||
+ row * prior_contexts_stride_0
|
||||
+ prior_step * prior_contexts_stride_1
|
||||
+ context_offset
|
||||
)
|
||||
matches &= context_token == prior_token
|
||||
repeated |= matches
|
||||
|
||||
for context_offset in range(CONTEXT_WIDTH):
|
||||
context_token = tl.load(
|
||||
contexts_ptr + row * context_stride + context_offset
|
||||
)
|
||||
tl.store(
|
||||
prior_contexts_ptr
|
||||
+ row * prior_contexts_stride_0
|
||||
+ step * prior_contexts_stride_1
|
||||
+ context_offset,
|
||||
context_token,
|
||||
)
|
||||
enabled = tl.load(enabled_ptr + row)
|
||||
repeated = enabled & (repeated == 0)
|
||||
tl.store(output_ptr + row, repeated)
|
||||
|
||||
|
||||
@@ -108,6 +212,7 @@ def _repeated_context_mask_cpu(
|
||||
max_history: int | None = None,
|
||||
include_prompt: bool = False,
|
||||
skip_partial_context: bool = False,
|
||||
history_offsets: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Reference implementation; the parity tests check the Triton kernel against it."""
|
||||
repeated = torch.zeros(len(req_indices), dtype=torch.bool)
|
||||
@@ -124,8 +229,11 @@ def _repeated_context_mask_cpu(
|
||||
history_tokens = all_token_ids[req_idx, sequence_start:total_len].tolist()
|
||||
prefix = [-1] * contexts.shape[-1]
|
||||
current_context = tuple(contexts[row].tolist())
|
||||
history_offset = 0 if history_offsets is None else int(history_offsets[row])
|
||||
history_start = (
|
||||
max(0, len(history_tokens) - max_history) if max_history is not None else 0
|
||||
max(0, len(history_tokens) - max(0, max_history - history_offset))
|
||||
if max_history is not None
|
||||
else 0
|
||||
)
|
||||
for history_pos, token_id in enumerate(history_tokens):
|
||||
if history_pos >= history_start and (
|
||||
@@ -146,6 +254,9 @@ def repeated_context_mask(
|
||||
max_history: int | None = None,
|
||||
include_prompt: bool = False,
|
||||
skip_partial_context: bool = False,
|
||||
history_offsets: torch.Tensor | None = None,
|
||||
local_positions: torch.Tensor | None = None,
|
||||
num_speculative_steps: int = 0,
|
||||
) -> torch.Tensor:
|
||||
"""Return, per row, whether the row's context already occurred in its history.
|
||||
|
||||
@@ -163,12 +274,23 @@ def repeated_context_mask(
|
||||
include_prompt: Search the prompt as well as the generated tokens.
|
||||
skip_partial_context: Mark contexts containing start padding so they use
|
||||
ordinary sampling.
|
||||
history_offsets: Number of newer, non-committed positions preceding each
|
||||
context. These positions count toward `max_history`. Must be 1-D
|
||||
over rows; a stride-0 broadcast row is fine.
|
||||
local_positions: Position within each request's speculative block. When
|
||||
provided, compare each context with earlier contexts in that block.
|
||||
num_speculative_steps: Maximum number of draft tokens in the block.
|
||||
|
||||
"""
|
||||
if max_history is not None and max_history < 1:
|
||||
raise ValueError("max_history must be positive or None")
|
||||
if (local_positions is None) != (num_speculative_steps == 0):
|
||||
raise ValueError(
|
||||
"local_positions and positive num_speculative_steps must be "
|
||||
"provided together"
|
||||
)
|
||||
if all_token_ids.device.type == "cpu":
|
||||
return _repeated_context_mask_cpu(
|
||||
repeated = _repeated_context_mask_cpu(
|
||||
all_token_ids,
|
||||
req_indices,
|
||||
prompt_lens,
|
||||
@@ -177,10 +299,42 @@ def repeated_context_mask(
|
||||
max_history,
|
||||
include_prompt,
|
||||
skip_partial_context,
|
||||
history_offsets,
|
||||
)
|
||||
if local_positions is not None:
|
||||
max_offset = num_speculative_steps + 1
|
||||
if max_history is not None:
|
||||
max_offset = min(max_offset, max_history + 1)
|
||||
for offset in range(1, max_offset):
|
||||
prior_contexts = torch.cat(
|
||||
(contexts[:offset], contexts[:-offset]), dim=0
|
||||
)
|
||||
prior_requests = torch.cat(
|
||||
(req_indices[:offset], req_indices[:-offset]), dim=0
|
||||
)
|
||||
prior_local_pos = torch.cat(
|
||||
(local_positions[:offset], local_positions[:-offset]), dim=0
|
||||
)
|
||||
repeated |= (
|
||||
(req_indices >= 0)
|
||||
& (local_positions >= offset)
|
||||
& (req_indices == prior_requests)
|
||||
& (local_positions == prior_local_pos + offset)
|
||||
& (contexts == prior_contexts).all(dim=-1)
|
||||
)
|
||||
if include_prompt:
|
||||
repeated |= (contexts < 0).any(dim=-1)
|
||||
return repeated
|
||||
|
||||
if contexts.stride(-1) != 1:
|
||||
contexts = contexts.contiguous()
|
||||
# Only history_offsets passes its row stride to the kernel.
|
||||
req_indices = req_indices.contiguous()
|
||||
if local_positions is not None:
|
||||
local_positions = local_positions.contiguous()
|
||||
# `prompt_lens` and `total_lens` are indexed flat by request slot, and
|
||||
# `all_token_ids` takes a row stride but is read flat within the row. Those
|
||||
# three are contiguous by contract; every caller owns them outright.
|
||||
repeated = torch.empty(len(req_indices), dtype=torch.bool, device=contexts.device)
|
||||
_repeated_context_mask_kernel[(len(req_indices),)](
|
||||
repeated,
|
||||
@@ -189,17 +343,104 @@ def repeated_context_mask(
|
||||
req_indices,
|
||||
prompt_lens,
|
||||
total_lens,
|
||||
history_offsets,
|
||||
0 if history_offsets is None else history_offsets.stride(0),
|
||||
contexts,
|
||||
contexts.stride(0),
|
||||
local_positions,
|
||||
None,
|
||||
0,
|
||||
0,
|
||||
None,
|
||||
0,
|
||||
None,
|
||||
CONTEXT_WIDTH=contexts.shape[-1],
|
||||
NUM_SPECULATIVE_STEPS=num_speculative_steps,
|
||||
MAX_HISTORY=0 if max_history is None else max_history,
|
||||
INCLUDE_PROMPT=include_prompt,
|
||||
SKIP_PARTIAL_CONTEXT=skip_partial_context,
|
||||
HAS_HISTORY_OFFSETS=history_offsets is not None,
|
||||
SPECULATIVE_CONTEXTS=local_positions is not None,
|
||||
DRAFT_CONTEXTS=False,
|
||||
BLOCK=512,
|
||||
)
|
||||
return repeated
|
||||
|
||||
|
||||
def draft_watermarking_mask(
|
||||
all_token_ids: torch.Tensor,
|
||||
req_indices: torch.Tensor,
|
||||
prompt_lens: torch.Tensor,
|
||||
total_lens: torch.Tensor,
|
||||
prior_contexts: torch.Tensor,
|
||||
contexts: torch.Tensor,
|
||||
steps: torch.Tensor,
|
||||
enabled: torch.Tensor,
|
||||
max_history: int | None,
|
||||
include_prompt: bool,
|
||||
) -> torch.Tensor:
|
||||
if contexts.device.type == "cpu":
|
||||
repeated = repeated_context_mask(
|
||||
all_token_ids,
|
||||
req_indices,
|
||||
prompt_lens,
|
||||
total_lens,
|
||||
contexts,
|
||||
max_history,
|
||||
include_prompt=include_prompt,
|
||||
history_offsets=steps,
|
||||
)
|
||||
prior_steps = torch.arange(prior_contexts.shape[1]).unsqueeze(0)
|
||||
in_history = prior_steps < steps.unsqueeze(1)
|
||||
if max_history is not None:
|
||||
in_history &= prior_steps >= steps.unsqueeze(1) - max_history
|
||||
repeated |= (
|
||||
(prior_contexts[: len(contexts)] == contexts.unsqueeze(1)).all(dim=-1)
|
||||
& in_history
|
||||
).any(dim=-1)
|
||||
if include_prompt:
|
||||
repeated |= (contexts < 0).any(dim=-1)
|
||||
row_indices = torch.arange(len(contexts))
|
||||
prior_contexts.index_put_((row_indices, steps), contexts)
|
||||
return enabled & ~repeated
|
||||
|
||||
if contexts.stride(-1) != 1:
|
||||
contexts = contexts.contiguous()
|
||||
# Only `steps` passes its row stride to the kernel.
|
||||
req_indices = req_indices.contiguous()
|
||||
enabled = enabled.contiguous()
|
||||
active = torch.empty(len(contexts), dtype=torch.bool, device=contexts.device)
|
||||
_repeated_context_mask_kernel[(len(contexts),)](
|
||||
active,
|
||||
all_token_ids,
|
||||
all_token_ids.stride(0),
|
||||
req_indices,
|
||||
prompt_lens,
|
||||
total_lens,
|
||||
steps,
|
||||
steps.stride(0),
|
||||
contexts,
|
||||
contexts.stride(0),
|
||||
None,
|
||||
prior_contexts,
|
||||
prior_contexts.stride(0),
|
||||
prior_contexts.stride(1),
|
||||
steps,
|
||||
steps.stride(0),
|
||||
enabled,
|
||||
NUM_SPECULATIVE_STEPS=prior_contexts.shape[1],
|
||||
CONTEXT_WIDTH=contexts.shape[1],
|
||||
MAX_HISTORY=0 if max_history is None else max_history,
|
||||
INCLUDE_PROMPT=include_prompt,
|
||||
SKIP_PARTIAL_CONTEXT=False,
|
||||
HAS_HISTORY_OFFSETS=True,
|
||||
SPECULATIVE_CONTEXTS=False,
|
||||
DRAFT_CONTEXTS=True,
|
||||
BLOCK=512,
|
||||
)
|
||||
return active
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _mulhilo32(multiplier: tl.constexpr, value):
|
||||
high = tl_math.umulhi(multiplier, value)
|
||||
@@ -351,28 +592,58 @@ def _philox_gumbel_kernel(
|
||||
seeds_ptr,
|
||||
pos_ptr,
|
||||
temp_ptr,
|
||||
logits_cache_ptr,
|
||||
logits_cache_stride_0,
|
||||
logits_cache_stride_1,
|
||||
logits_cache_col_ptr,
|
||||
logits_cache_source_ptr,
|
||||
logits_cache_source_stride,
|
||||
key_0_value,
|
||||
key_1_value,
|
||||
vocab_size,
|
||||
CONTEXT_WIDTH: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
USE_FP64: tl.constexpr,
|
||||
IS_DRAFTING: tl.constexpr,
|
||||
PER_TOKEN_COL: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
block_index = tl.program_id(1)
|
||||
groups = block_index * (BLOCK_SIZE // 4) + tl.arange(0, BLOCK_SIZE // 4)
|
||||
candidate = block_index * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = candidate < vocab_size
|
||||
|
||||
if skip_mask_ptr is not None:
|
||||
req_state_idx = row
|
||||
valid_req = True
|
||||
if expanded_idx_mapping_ptr is not None:
|
||||
req_state_idx = tl.load(expanded_idx_mapping_ptr + row).to(tl.int64)
|
||||
valid_req = req_state_idx >= 0
|
||||
|
||||
if logits_cache_ptr is not None:
|
||||
if PER_TOKEN_COL:
|
||||
col = tl.load(logits_cache_col_ptr + row)
|
||||
else:
|
||||
col = tl.load(logits_cache_col_ptr)
|
||||
cached_logits = tl.load(
|
||||
logits_cache_source_ptr + row * logits_cache_source_stride + candidate,
|
||||
mask=mask,
|
||||
)
|
||||
tl.store(
|
||||
logits_cache_ptr
|
||||
+ req_state_idx * logits_cache_stride_0
|
||||
+ col * logits_cache_stride_1
|
||||
+ candidate,
|
||||
cached_logits,
|
||||
mask=mask & valid_req,
|
||||
)
|
||||
|
||||
if skip_mask_ptr is not None: # noqa: SIM102
|
||||
if tl.load(skip_mask_ptr + row):
|
||||
temp = tl.load(temp_ptr + req_state_idx, mask=valid_req, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
seed = tl.load(seeds_ptr + req_state_idx, mask=valid_req, other=0)
|
||||
pos = tl.load(pos_ptr + row)
|
||||
candidate = block_index * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = candidate < vocab_size
|
||||
logits_row = logits_ptr + row * logits_stride
|
||||
logits = tl.load(logits_row + candidate, mask=mask, other=float("-inf")).to(
|
||||
tl.float32
|
||||
@@ -384,7 +655,7 @@ def _philox_gumbel_kernel(
|
||||
seed,
|
||||
pos,
|
||||
temp,
|
||||
IS_DRAFTING=False,
|
||||
IS_DRAFTING=IS_DRAFTING,
|
||||
USE_FP64=USE_FP64,
|
||||
APPLY_TEMPERATURE=False,
|
||||
)
|
||||
@@ -443,12 +714,37 @@ def philox_gumbel_sample(
|
||||
seeds: torch.Tensor | None = None,
|
||||
positions: torch.Tensor | None = None,
|
||||
use_fp64: bool = False,
|
||||
is_drafting: bool = False,
|
||||
logits_cache: torch.Tensor | None = None,
|
||||
logits_cache_col: torch.Tensor | None = None,
|
||||
logits_cache_source: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
sampling_state = (expanded_idx_mapping, temperatures, seeds, positions)
|
||||
if skip_mask is None and any(value is not None for value in sampling_state):
|
||||
raise ValueError("sampling state requires skip_mask")
|
||||
if skip_mask is not None and any(value is None for value in sampling_state):
|
||||
raise ValueError("skip_mask requires complete sampling state")
|
||||
if logits_cache is not None:
|
||||
if logits_cache_col is None or logits_cache_source is None:
|
||||
raise ValueError("logits cache requires its column and source logits")
|
||||
if expanded_idx_mapping is None:
|
||||
raise ValueError("logits cache requires expanded_idx_mapping")
|
||||
assert logits_cache_source.shape == logits.shape, (
|
||||
"logits cache source must match sampled logits shape"
|
||||
)
|
||||
assert logits_cache_source.device == logits.device, (
|
||||
"logits cache source must be on the sampled logits device"
|
||||
)
|
||||
assert logits_cache_source.dtype == logits_cache.dtype, (
|
||||
"logits cache source and destination must have the same dtype"
|
||||
)
|
||||
assert logits_cache.size(-1) >= logits.shape[-1], (
|
||||
f"draft logits cache vocab dim ({logits_cache.size(-1)}) is narrower "
|
||||
f"than the sampled logits ({logits.shape[-1]}). Cached logits would be "
|
||||
"truncated."
|
||||
)
|
||||
elif logits_cache_col is not None or logits_cache_source is not None:
|
||||
raise ValueError("logits cache column and source require logits_cache")
|
||||
|
||||
if logits.stride(-1) != 1:
|
||||
logits = logits.contiguous()
|
||||
@@ -460,6 +756,10 @@ def philox_gumbel_sample(
|
||||
expanded_idx_mapping = expanded_idx_mapping.contiguous()
|
||||
if positions is not None:
|
||||
positions = positions.contiguous()
|
||||
if logits_cache_col is not None:
|
||||
logits_cache_col = logits_cache_col.contiguous()
|
||||
if logits_cache_source is not None and logits_cache_source.stride(-1) != 1:
|
||||
logits_cache_source = logits_cache_source.contiguous()
|
||||
num_tokens, vocab_size = logits.shape
|
||||
block_size = 1024
|
||||
num_blocks = triton.cdiv(vocab_size, block_size)
|
||||
@@ -483,12 +783,20 @@ def philox_gumbel_sample(
|
||||
seeds,
|
||||
positions,
|
||||
temperatures,
|
||||
logits_cache,
|
||||
logits_cache.stride(0) if logits_cache is not None else 0,
|
||||
logits_cache.stride(1) if logits_cache is not None else 0,
|
||||
logits_cache_col,
|
||||
logits_cache_source,
|
||||
logits_cache_source.stride(0) if logits_cache_source is not None else 0,
|
||||
key & _UINT32_MASK_VALUE,
|
||||
key >> 32,
|
||||
vocab_size,
|
||||
CONTEXT_WIDTH=contexts.shape[-1],
|
||||
BLOCK_SIZE=block_size,
|
||||
USE_FP64=use_fp64,
|
||||
IS_DRAFTING=is_drafting,
|
||||
PER_TOKEN_COL=logits_cache_col is not None and logits_cache_col.dim() > 0,
|
||||
)
|
||||
max_block_index = local_max.argmax(dim=-1, keepdim=True)
|
||||
return local_argmax.gather(dim=-1, index=max_block_index).view(-1)
|
||||
|
||||
@@ -135,7 +135,12 @@ class DSparkSpeculator(DFlashSpeculator):
|
||||
|
||||
# sample_pos is the predicted token's position P. Sampling keys a draw
|
||||
# by the position before the sampled token, P-1.
|
||||
sampled = gumbel_sample(
|
||||
sampler = (
|
||||
gumbel_sample
|
||||
if self.draft_watermarker is None
|
||||
else self.draft_watermarker.sample
|
||||
)
|
||||
return sampler(
|
||||
logits,
|
||||
idx_map,
|
||||
self.temperature,
|
||||
@@ -147,11 +152,6 @@ class DSparkSpeculator(DFlashSpeculator):
|
||||
logits_cache_col=self._step_cols[step],
|
||||
use_fp64=self.use_fp64_gumbel,
|
||||
)
|
||||
if self.draft_watermarker is not None:
|
||||
sampled = self.draft_watermarker.sample(
|
||||
logits, sampled, idx_map, self.temperature
|
||||
)
|
||||
return sampled
|
||||
|
||||
def _sample_sequential(self, num_reqs: int, head_hidden: torch.Tensor) -> None:
|
||||
if self._draft_topk is not None:
|
||||
|
||||
@@ -226,11 +226,18 @@ class RejectionSampler:
|
||||
if self.watermark_key is None:
|
||||
return {}
|
||||
assert isinstance(self.sampler, GPUWatermarkSampler)
|
||||
contexts = self.sampler._get_contexts(
|
||||
expanded_idx_mapping, expanded_local_pos, draft_sampled
|
||||
)
|
||||
watermarking_skip_mask = None
|
||||
if self.sampler.deduplicate_contexts != "none":
|
||||
watermarking_skip_mask = self.sampler._get_repeated_contexts(
|
||||
expanded_idx_mapping, contexts, expanded_local_pos
|
||||
)
|
||||
return {
|
||||
"contexts": self.sampler._get_contexts(
|
||||
expanded_idx_mapping, expanded_local_pos, draft_sampled
|
||||
),
|
||||
"contexts": contexts,
|
||||
"watermarking": self.sampler.watermarking.gpu,
|
||||
"watermarking_skip_mask": watermarking_skip_mask,
|
||||
"watermark_key": self.watermark_key,
|
||||
}
|
||||
|
||||
|
||||
@@ -564,7 +564,6 @@ def _rejection_kernel(
|
||||
# stores the target argmax upon first rejection, so it rejects the
|
||||
# placeholder via `accepted` instead.
|
||||
verifying &= is_valid_draft
|
||||
|
||||
if verifying:
|
||||
pos = tl.load(pos_ptr + logit_idx)
|
||||
u = tl_rand32(seed, pos, includes_zero=False)
|
||||
@@ -719,6 +718,8 @@ def _seeded_resample_argmax(
|
||||
0, # logits_cache_stride_0
|
||||
0, # logits_cache_stride_1
|
||||
None, # logits_cache_col_ptr
|
||||
None, # logits_cache_source_ptr
|
||||
0, # logits_cache_source_stride
|
||||
vocab_size,
|
||||
IS_DRAFTING=False,
|
||||
APPLY_TEMPERATURE=False,
|
||||
@@ -766,6 +767,7 @@ def _resample_kernel(
|
||||
contexts_stride,
|
||||
# [max_num_reqs], uint8 view of a bool tensor
|
||||
watermarking_ptr,
|
||||
watermarking_skip_mask_ptr,
|
||||
watermark_key_0,
|
||||
watermark_key_1,
|
||||
vocab_size,
|
||||
@@ -775,6 +777,7 @@ def _resample_kernel(
|
||||
USE_BLOCK_VERIFICATION: tl.constexpr,
|
||||
CONTEXT_WIDTH: tl.constexpr,
|
||||
WATERMARK: tl.constexpr,
|
||||
DEDUPLICATE_CONTEXTS: tl.constexpr,
|
||||
):
|
||||
req_idx = tl.program_id(0)
|
||||
resample_idx = tl.load(rejected_step_ptr + req_idx)
|
||||
@@ -807,6 +810,25 @@ def _resample_kernel(
|
||||
).to(tl.float32)
|
||||
|
||||
# Compute the residual logits to resample the rejected token from.
|
||||
is_watermarked = False
|
||||
if WATERMARK:
|
||||
is_watermarked = (
|
||||
tl.load(
|
||||
watermarking_ptr + req_state_idx,
|
||||
mask=req_state_idx >= 0,
|
||||
other=0,
|
||||
)
|
||||
!= 0
|
||||
)
|
||||
|
||||
skip_watermarking = False
|
||||
if DEDUPLICATE_CONTEXTS:
|
||||
skip_watermarking = (
|
||||
(tl.load(watermarking_skip_mask_ptr + resample_token_idx) != 0)
|
||||
& is_watermarked
|
||||
& (temp != 0.0)
|
||||
)
|
||||
|
||||
if is_bonus or not is_valid_rejected_draft:
|
||||
# Bonus token (no rejections) or -1 placeholder token. In either case,
|
||||
# directly use the target logits.
|
||||
@@ -868,15 +890,7 @@ def _resample_kernel(
|
||||
# Resample the rejected/bonus token.
|
||||
if WATERMARK:
|
||||
# Padded and greedy rows retain the stock draw.
|
||||
is_watermarked = (
|
||||
tl.load(
|
||||
watermarking_ptr + req_state_idx,
|
||||
mask=req_state_idx >= 0,
|
||||
other=0,
|
||||
)
|
||||
!= 0
|
||||
)
|
||||
if is_watermarked & (temp != 0.0):
|
||||
if is_watermarked & (temp != 0.0) & ~skip_watermarking:
|
||||
watermark_value, idx = philox_gumbel_block_argmax(
|
||||
residual_logits,
|
||||
mask,
|
||||
@@ -1023,6 +1037,7 @@ def rejection_sample(
|
||||
use_block_verification: bool = False,
|
||||
contexts: torch.Tensor | None = None,
|
||||
watermarking: torch.Tensor | None = None,
|
||||
watermarking_skip_mask: torch.Tensor | None = None,
|
||||
watermark_key: int | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert target_logits.ndim == 2 and target_logits.stride(-1) == 1
|
||||
@@ -1036,9 +1051,13 @@ def rejection_sample(
|
||||
assert watermark == (watermarking is not None) == (watermark_key is not None), (
|
||||
"contexts, watermarking and watermark_key must be set together."
|
||||
)
|
||||
assert watermarking_skip_mask is None or watermark, (
|
||||
"watermarking_skip_mask requires watermarking."
|
||||
)
|
||||
contexts_stride = 0
|
||||
context_width = 1
|
||||
watermarking_bytes: torch.Tensor | None = None
|
||||
watermarking_skip_mask_bytes: torch.Tensor | None = None
|
||||
watermark_key_0 = 0
|
||||
watermark_key_1 = 0
|
||||
if contexts is not None:
|
||||
@@ -1053,6 +1072,12 @@ def rejection_sample(
|
||||
contexts_stride = contexts.stride(0)
|
||||
context_width = contexts.shape[-1]
|
||||
watermarking_bytes = watermarking.view(torch.uint8)
|
||||
if watermarking_skip_mask is not None:
|
||||
assert watermarking_skip_mask.shape == (num_logits,)
|
||||
assert watermarking_skip_mask.dtype == torch.bool
|
||||
if not watermarking_skip_mask.is_contiguous():
|
||||
watermarking_skip_mask = watermarking_skip_mask.contiguous()
|
||||
watermarking_skip_mask_bytes = watermarking_skip_mask.view(torch.uint8)
|
||||
watermark_key_0 = watermark_key & 0xFFFFFFFF
|
||||
watermark_key_1 = watermark_key >> 32
|
||||
draft_logits_stride_0 = 0
|
||||
@@ -1269,6 +1294,7 @@ def rejection_sample(
|
||||
contexts,
|
||||
contexts_stride,
|
||||
watermarking_bytes,
|
||||
watermarking_skip_mask_bytes,
|
||||
watermark_key_0,
|
||||
watermark_key_1,
|
||||
vocab_size,
|
||||
@@ -1278,6 +1304,7 @@ def rejection_sample(
|
||||
USE_BLOCK_VERIFICATION=use_block_verification,
|
||||
CONTEXT_WIDTH=context_width,
|
||||
WATERMARK=watermark,
|
||||
DEDUPLICATE_CONTEXTS=watermarking_skip_mask is not None,
|
||||
)
|
||||
|
||||
# Insert the resampled tokens into the output sampled.
|
||||
|
||||
@@ -38,6 +38,7 @@ from vllm.v1.worker.gpu.spec_decode.acceptance_estimator import (
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.watermarking.gpu_sampler import GPUWatermarkSampler
|
||||
from vllm.v1.worker.gpu.pcp_manager import PCPManager
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -198,6 +199,9 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
self.max_num_reqs,
|
||||
device,
|
||||
watermark_config.allow_target_only_watermarking,
|
||||
self.num_speculative_steps,
|
||||
watermark_config.deduplicate_contexts,
|
||||
watermark_config.deduplicate_contexts_max_history,
|
||||
)
|
||||
|
||||
self.supports_mm_inputs = False
|
||||
@@ -411,7 +415,12 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
) -> torch.Tensor:
|
||||
if draft_logits is not None:
|
||||
logits = self.model.compute_logits(hidden_states)
|
||||
sampled = gumbel_sample(
|
||||
sampler = (
|
||||
gumbel_sample
|
||||
if self.draft_watermarker is None
|
||||
else self.draft_watermarker.sample
|
||||
)
|
||||
sampled = sampler(
|
||||
logits,
|
||||
idx_mapping,
|
||||
temperature,
|
||||
@@ -423,10 +432,6 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
logits_cache_col=draft_step,
|
||||
use_fp64=self.use_fp64_gumbel,
|
||||
)
|
||||
if self.draft_watermarker is not None:
|
||||
sampled = self.draft_watermarker.sample(
|
||||
logits, sampled, idx_mapping, temperature
|
||||
)
|
||||
elif self.use_local_argmax_reduction:
|
||||
return self.model.get_top_tokens(hidden_states)
|
||||
else:
|
||||
@@ -462,11 +467,19 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
self.acceptance_estimator.step(idx_mapping, num_sampled, num_rejected)
|
||||
|
||||
def prepare_watermarking(
|
||||
self, contexts: torch.Tensor, watermarking: torch.Tensor
|
||||
self,
|
||||
sampler: "GPUWatermarkSampler",
|
||||
idx_mapping: torch.Tensor,
|
||||
) -> None:
|
||||
if self.draft_watermarker is None:
|
||||
return
|
||||
self.draft_watermarker.prepare(contexts, watermarking)
|
||||
self.draft_watermarker.prepare(
|
||||
sampler._get_contexts(idx_mapping),
|
||||
sampler.watermarking.gpu[idx_mapping],
|
||||
sampler.req_states.all_token_ids.gpu,
|
||||
sampler.req_states.prompt_len.gpu,
|
||||
sampler.req_states.total_len.gpu,
|
||||
)
|
||||
|
||||
def _copy_request_inputs(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user