[watermarking] Dual-key gumbel-max watermarking for speculative decoding support (#56122)

Signed-off-by: Raphael Rialland <raphael.rialland@mistral.ai>
Signed-off-by: Simon Veitner <sveitner@redhat.com>
Signed-off-by: Tomas Ruiz <tomas.ruiz.te@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Simon Veitner <sveitner@redhat.com>
Co-authored-by: Tomas Ruiz <tomas.ruiz.te@gmail.com>
This commit is contained in:
Raphaël Rialland
2026-09-12 18:42:01 +00:00
committed by GitHub
co-authored by OpenAI Codex Simon Veitner Tomas Ruiz
parent 1ee4be4dbc
commit 7ee8a6dd01
28 changed files with 3460 additions and 245 deletions
+22 -3
View File
@@ -53,18 +53,37 @@ def test_watermark_config_cli():
"--model",
"dummy",
"--watermark-config",
'{"algorithm":"gumbel","key":42,"prf":"philox"}',
'{"algorithm":"dual_key_gumbel","key":42,"prf":"philox","alpha":0.25,'
'"allow_target_only_watermarking":true}',
]
)
config = EngineArgs.from_cli_args(args).create_watermark_config()
assert config is not None
assert config.algorithm == "gumbel"
assert config.algorithm == "dual_key_gumbel"
assert config.key == 42
assert config.alpha == 0.25
assert config.context_width == 4
assert config.deduplicate_contexts == "single_turn"
assert config.deduplicate_contexts_max_history == 8192
assert config.prf == "philox"
assert not config.supports_speculative_decoding
assert config.allow_target_only_watermarking
args = parser.parse_args(
[
"--model",
"dummy",
"--watermark-config",
'{"key":42,"deduplicate_contexts":"none",'
'"deduplicate_contexts_max_history":32}',
]
)
config = EngineArgs.from_cli_args(args).create_watermark_config()
assert config is not None
assert config.deduplicate_contexts == "none"
assert config.deduplicate_contexts_max_history == 32
@pytest.mark.parametrize(
@@ -0,0 +1,29 @@
model_name: "Qwen/Qwen3-4B-FP8"
num_questions: 1319
num_fewshot: 5
max_tokens: 256
use_chat_completions: false
temperature: 1.0
seed: 42
# Reference accuracy from the non-watermarked DSpark regression test.
accuracy_threshold: 0.801
tolerance: 0.02
# Measured TP1 acceptance length: 4.115; allow a 0.2 regression.
min_acceptance_length: 3.915
env:
VLLM_USE_V2_MODEL_RUNNER: "1"
server_args: >-
--tensor-parallel-size 2
--max-model-len 4096
--gpu-memory-utilization 0.92
--no-enable-prefix-caching
--speculative-config.method dspark
--speculative-config.model deepseek-ai/dspark_qwen3_4b_block7
--speculative-config.num_speculative_tokens 7
--speculative-config.draft_sample_method probabilistic
--watermark-config.algorithm dual_key_gumbel
--watermark-config.key 42
--watermark-config.context_width 4
--watermark-config.deduplicate_contexts none
@@ -0,0 +1 @@
Qwen3-4B-FP8-DSpark-watermark-TP2.yaml
@@ -0,0 +1,121 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from vllm.config.watermarking import WatermarkConfig, derive_watermark_key
from vllm.model_executor.warmup.spec_decode_rejection_warmup import (
spec_decode_rejection_warmup,
)
MASTER_KEY = 20250910
def _worker(
watermark_config: WatermarkConfig | None,
draft_sample_method: str = "greedy",
) -> SimpleNamespace:
return SimpleNamespace(
device=torch.device("cpu"),
vllm_config=SimpleNamespace(
speculative_config=SimpleNamespace(
num_speculative_tokens=2,
rejection_sample_method="standard",
draft_sample_method=draft_sample_method,
),
model_config=SimpleNamespace(
get_vocab_size=lambda: 128, dtype=torch.bfloat16
),
watermark_config=watermark_config,
),
)
def _capture_calls(monkeypatch: pytest.MonkeyPatch) -> list[dict]:
calls: list[dict] = []
def fake_rejection_sample(**kwargs):
calls.append(kwargs)
return None, None
monkeypatch.setattr(
"vllm.v1.worker.gpu.spec_decode.rejection_sampler_utils.rejection_sample",
fake_rejection_sample,
)
return calls
def test_warmup_token_id_dtypes_match_the_runtime_buffers(
monkeypatch: pytest.MonkeyPatch,
) -> None:
calls = _capture_calls(monkeypatch)
spec_decode_rejection_warmup(_worker(None))
assert calls
for kwargs in calls:
assert kwargs["draft_sampled"].dtype == torch.int32
assert kwargs["pos"].dtype == torch.int64
def test_warmup_index_mapping_dtypes_match_the_runtime_buffers(
monkeypatch: pytest.MonkeyPatch,
) -> None:
calls = _capture_calls(monkeypatch)
spec_decode_rejection_warmup(_worker(None))
assert calls
for kwargs in calls:
assert kwargs["idx_mapping"].dtype == torch.int64
assert kwargs["expanded_idx_mapping"].dtype == torch.int64
assert kwargs["expanded_local_pos"].dtype == torch.int32
def test_warmup_watermark_args_match_the_runtime_kernel_arguments(
monkeypatch: pytest.MonkeyPatch,
) -> None:
watermark_config = WatermarkConfig(
key=MASTER_KEY, algorithm="dual_key_gumbel", context_width=3
)
calls = _capture_calls(monkeypatch)
spec_decode_rejection_warmup(
_worker(watermark_config, draft_sample_method="probabilistic")
)
assert calls
key_b = derive_watermark_key(MASTER_KEY, b"key_b")
for kwargs in calls:
contexts = kwargs["contexts"]
assert contexts.dtype == torch.int32
assert contexts.shape == (3, 3) and contexts.stride() == (3, 1)
assert kwargs["watermarking"].dtype == torch.bool
assert kwargs["watermark_key"] == key_b
assert key_b & 0xFFFFFFFF >= 2**31 and key_b >> 32 >= 2**31
def test_warmup_is_skipped_when_the_recovery_key_cannot_be_derived(
monkeypatch: pytest.MonkeyPatch,
) -> None:
calls = _capture_calls(monkeypatch)
monkeypatch.setattr(
"vllm.v1.watermarking.spec_decode.speculative_target_watermark_key",
lambda _: (_ for _ in ()).throw(NotImplementedError("no philox")),
)
spec_decode_rejection_warmup(_worker(WatermarkConfig(key=MASTER_KEY)))
assert calls == []
def test_warmup_draft_logits_match_the_runtime_draft_sample_method(
monkeypatch: pytest.MonkeyPatch,
) -> None:
calls = _capture_calls(monkeypatch)
spec_decode_rejection_warmup(_worker(None))
assert calls
assert all(kwargs["draft_logits"] is None for kwargs in calls)
calls = _capture_calls(monkeypatch)
spec_decode_rejection_warmup(_worker(None, draft_sample_method="probabilistic"))
assert calls
assert all(kwargs["draft_logits"] is not None for kwargs in calls)
@@ -0,0 +1,300 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The warmup must cover every specialization the sampler will launch.
``_philox_gumbel_kernel`` is keyed on the Philox key halves, on the logits
dtype (model dtype when the sampler passes logits through, fp32 when logits
processing copies them) and on whether a skip mask is passed, so a missing
combination leaves a request paying the compile this module exists to remove.
"""
import logging
from types import SimpleNamespace
import pytest
import torch
from vllm.config.watermarking import WatermarkConfig, derive_watermark_key
from vllm.model_executor.warmup.watermark_sample_warmup import watermark_sample_warmup
from vllm.platforms import current_platform
from vllm.v1.watermarking.factory import create_watermarker
from vllm.v1.watermarking.gpu_sampler import GPUWatermarkSampler
from vllm.v1.watermarking.spec_decode import create_speculative_target_watermarker
MASTER_KEY = 20250910
KEY_A = derive_watermark_key(MASTER_KEY, b"key_a")
KEY_B = derive_watermark_key(MASTER_KEY, b"key_b")
@pytest.fixture(autouse=True)
def cuda_alike_platform(monkeypatch: pytest.MonkeyPatch) -> None:
"""The warmup returns early off CUDA, so pin the platform it reads.
Everything here runs on the fake worker's cpu device; without this the
tests below would pass only on a CUDA-alike host and silently warm nothing
anywhere else.
"""
monkeypatch.setattr(current_platform, "is_cuda_alike", lambda: True)
def _sampler(
watermark_config: WatermarkConfig,
speculative: bool,
use_fp64_gumbel: bool,
) -> GPUWatermarkSampler:
"""The sampler ``GPUModelRunner`` builds for this config.
Mirrors ``vllm/v1/worker/gpu/model_runner.py:473-486``. ``__new__`` skips
``Sampler.__init__``, which allocates UVA-backed device buffers; the warmup
only reads the three attributes set here.
"""
watermarker = create_watermarker(watermark_config)
if speculative:
watermarker = create_speculative_target_watermarker(watermarker)
sampler = GPUWatermarkSampler.__new__(GPUWatermarkSampler)
sampler.watermarker = watermarker
sampler.deduplicate_contexts = watermark_config.deduplicate_contexts
sampler.use_fp64_gumbel = use_fp64_gumbel
return sampler
def _worker(
watermark_config: WatermarkConfig | None,
speculative: bool = False,
use_fp64_gumbel: bool = False,
with_sampler: bool = True,
) -> SimpleNamespace:
sampler = (
_sampler(watermark_config, speculative, use_fp64_gumbel)
if watermark_config is not None and with_sampler
else None
)
return SimpleNamespace(
device=torch.device("cpu"),
model_runner=SimpleNamespace(sampler=sampler),
vllm_config=SimpleNamespace(
model_config=SimpleNamespace(
get_vocab_size=lambda: 128,
dtype=torch.bfloat16,
),
watermark_config=watermark_config,
),
)
def _capture_calls(monkeypatch: pytest.MonkeyPatch) -> list[dict]:
calls: list[dict] = []
def fake_philox_gumbel_sample(logits, contexts, key, **kwargs):
calls.append({"logits": logits, "contexts": contexts, "key": key, **kwargs})
return torch.zeros(logits.shape[0], dtype=torch.int64)
monkeypatch.setattr(
"vllm.v1.worker.gpu.sample.watermark.philox_gumbel_sample",
fake_philox_gumbel_sample,
)
return calls
def _variants(calls: list[dict]) -> set[tuple[int, torch.dtype, bool]]:
return {
(call["key"], call["logits"].dtype, call.get("skip_mask") is not None)
for call in calls
}
def test_no_warmup_without_a_watermark_config(
monkeypatch: pytest.MonkeyPatch,
) -> None:
calls = _capture_calls(monkeypatch)
watermark_sample_warmup(_worker(None))
assert calls == []
def test_no_warmup_without_a_watermarked_sampler(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A rank that does not sample launches nothing to warm.
Only the final pipeline stage of a generative model holds a
``GPUWatermarkSampler``; on any other rank the kernel is never launched.
"""
calls = _capture_calls(monkeypatch)
watermark_sample_warmup(
_worker(WatermarkConfig(key=MASTER_KEY), with_sampler=False)
)
assert calls == []
def test_no_warmup_off_cuda(monkeypatch: pytest.MonkeyPatch) -> None:
"""``GumbelWatermarker`` launches the kernel only on a CUDA-alike device.
Elsewhere -- an XPU worker inherits ``kernel_warmup`` from the CUDA worker
-- the sampler takes the torch path, so warming would compile a kernel
nothing launches.
"""
calls = _capture_calls(monkeypatch)
monkeypatch.setattr(current_platform, "is_cuda_alike", lambda: False)
watermark_sample_warmup(_worker(WatermarkConfig(key=MASTER_KEY)))
assert calls == []
def test_single_key_warms_both_logits_dtypes(
monkeypatch: pytest.MonkeyPatch,
) -> None:
calls = _capture_calls(monkeypatch)
watermark_sample_warmup(_worker(WatermarkConfig(key=MASTER_KEY)))
assert _variants(calls) == {
(MASTER_KEY, torch.bfloat16, True),
(MASTER_KEY, torch.float32, True),
}
assert len(calls) == len(_variants(calls))
def test_dual_key_warms_both_derived_keys(monkeypatch: pytest.MonkeyPatch) -> None:
"""``DualKeyGumbelWatermarker.sample`` launches the kernel once per key.
The halves of a derived key exceed 2**31, so each key is typed i64/i32 by
magnitude and a wrong key compiles a specialization the engine never uses.
"""
calls = _capture_calls(monkeypatch)
watermark_sample_warmup(
_worker(WatermarkConfig(key=MASTER_KEY, algorithm="dual_key_gumbel"))
)
assert _variants(calls) == {
(KEY_A, torch.bfloat16, True),
(KEY_A, torch.float32, True),
(KEY_B, torch.bfloat16, True),
(KEY_B, torch.float32, True),
}
@pytest.mark.parametrize("alpha,key", [(0.0, KEY_A), (1.0, KEY_B)])
def test_dual_key_alpha_bounds_warm_one_key(
monkeypatch: pytest.MonkeyPatch, alpha: float, key: int
) -> None:
calls = _capture_calls(monkeypatch)
watermark_sample_warmup(
_worker(
WatermarkConfig(key=MASTER_KEY, algorithm="dual_key_gumbel", alpha=alpha)
)
)
assert _variants(calls) == {
(key, torch.bfloat16, True),
(key, torch.float32, True),
}
def test_speculative_config_warms_the_target_role_only(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Under spec decode the sampler holds ``create_target_watermarker()``.
Rows without draft tokens sample through it, so the sampler path launches
key B alone; the draft key runs in the speculator's own sampler.
"""
calls = _capture_calls(monkeypatch)
watermark_sample_warmup(
_worker(
WatermarkConfig(key=MASTER_KEY, algorithm="dual_key_gumbel"),
speculative=True,
)
)
assert _variants(calls) == {
(KEY_B, torch.bfloat16, True),
(KEY_B, torch.float32, True),
}
def test_dedup_none_also_warms_the_mask_free_specialization(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Without deduplication an all-watermarked batch passes no skip mask.
A later batch mixing in an opted-out request passes one, so both
specializations are reachable and both must be warmed.
"""
calls = _capture_calls(monkeypatch)
watermark_sample_warmup(
_worker(WatermarkConfig(key=MASTER_KEY, deduplicate_contexts="none"))
)
assert _variants(calls) == {
(MASTER_KEY, torch.bfloat16, True),
(MASTER_KEY, torch.float32, True),
(MASTER_KEY, torch.bfloat16, False),
(MASTER_KEY, torch.float32, False),
}
for call in calls:
if call.get("skip_mask") is None:
assert "temperatures" not in call
def test_warmup_argument_dtypes_match_the_runtime_buffers(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Contexts are int32, the sampling state int64 indices and fp32 temperatures.
``GPUWatermarkSampler._get_contexts`` gathers from the int32 request-state
token ids, ``expanded_idx_mapping``/``positions``/``seeds`` are int64 and
``SamplingStates.temperature`` is fp32; warming other dtypes compiles
specializations no engine launches.
"""
calls = _capture_calls(monkeypatch)
watermark_sample_warmup(_worker(WatermarkConfig(key=MASTER_KEY, context_width=3)))
assert calls
for call in calls:
num_tokens = call["logits"].shape[0]
assert call["contexts"].dtype == torch.int32
assert call["contexts"].shape == (num_tokens, 3)
assert call["skip_mask"].dtype == torch.bool
assert call["expanded_idx_mapping"].dtype == torch.int64
assert call["positions"].dtype == torch.int64
assert call["seeds"].dtype == torch.int64
assert call["temperatures"].dtype == torch.float32
def test_use_fp64_follows_the_path_that_forwards_it(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Only the mixed path forwards the sampler's ``use_fp64_gumbel``.
``GumbelWatermarker._sample_watermarked`` launches the mask-free variant
with the ``use_fp64=False`` default, so warming it with fp64 would compile
a specialization the engine never launches and miss the one it does.
"""
calls = _capture_calls(monkeypatch)
watermark_sample_warmup(
_worker(
WatermarkConfig(key=MASTER_KEY, deduplicate_contexts="none"),
use_fp64_gumbel=True,
)
)
assert calls
for call in calls:
assert call["use_fp64"] is (call.get("skip_mask") is not None)
def test_warmup_failure_does_not_abort_startup(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
def failing_philox_gumbel_sample(*args, **kwargs):
raise RuntimeError("no triton")
monkeypatch.setattr(
"vllm.v1.worker.gpu.sample.watermark.philox_gumbel_sample",
failing_philox_gumbel_sample,
)
with caplog.at_level(logging.WARNING):
watermark_sample_warmup(_worker(WatermarkConfig(key=MASTER_KEY)))
assert "Skipping watermark sampler warmup." in caplog.text
+180 -5
View File
@@ -2856,14 +2856,183 @@ def _watermarked_vllm_config() -> VllmConfig:
return config
def test_gumbel_watermark_rejects_speculative_decoding():
def test_target_only_gumbel_allows_speculative_decoding(caplog_vllm, disable_log_dedup):
config = _watermarked_vllm_config()
config.speculative_config = SpeculativeConfig(
method="ngram",
num_speculative_tokens=1,
config.watermark_config = WatermarkConfig(
key=42, allow_target_only_watermarking=True
)
config.speculative_config = SimpleNamespace(
method="mtp",
draft_sample_method="probabilistic",
rejection_sample_method="standard",
parallel_drafting=False,
)
with pytest.raises(ValueError, match="does not support speculative decoding"):
with caplog_vllm.at_level(logging.WARNING):
config._check_watermarking_unsupported()
assert "Target-only watermarking leaves accepted draft tokens" in caplog_vllm.text
assert "Context deduplication is not supported" in caplog_vllm.text
def test_speculative_watermarking_without_context_dedup_does_not_warn(
caplog_vllm, disable_log_dedup
):
config = _watermarked_vllm_config()
config.watermark_config = WatermarkConfig(
algorithm="dual_key_gumbel", key=42, deduplicate_contexts="none"
)
config.speculative_config = SimpleNamespace(
method="mtp",
draft_sample_method="probabilistic",
rejection_sample_method="standard",
parallel_drafting=False,
)
with caplog_vllm.at_level(logging.WARNING):
config._check_watermarking_unsupported()
assert "Context deduplication is not supported" not in caplog_vllm.text
def test_gumbel_rejects_speculative_decoding_without_target_only():
config = _watermarked_vllm_config()
config.watermark_config = WatermarkConfig(
algorithm="gumbel", key=42, allow_target_only_watermarking=False
)
config.speculative_config = SimpleNamespace(
method="mtp",
draft_sample_method="probabilistic",
rejection_sample_method="standard",
parallel_drafting=False,
)
with pytest.raises(ValueError, match="'gumbel'.*allow_target_only_watermarking"):
config._check_watermarking_unsupported()
def test_dual_key_gumbel_warns_that_configured_alpha_is_unused(
caplog_vllm, disable_log_dedup
):
config = _watermarked_vllm_config()
config.watermark_config = WatermarkConfig(
algorithm="dual_key_gumbel", key=42, alpha=0.25
)
config.speculative_config = SimpleNamespace(
method="mtp",
draft_sample_method="probabilistic",
rejection_sample_method="standard",
parallel_drafting=False,
)
with caplog_vllm.at_level(logging.WARNING):
config._check_watermarking_unsupported()
assert "The configured alpha=0.25 is not used" in caplog_vllm.text
@pytest.mark.parametrize(
("alpha", "speculative"),
[(0.1, True), (0.25, False)],
ids=["default-alpha", "no-specdec"],
)
def test_dual_key_gumbel_alpha_warning_is_not_emitted(
caplog_vllm, disable_log_dedup, alpha, speculative
):
config = _watermarked_vllm_config()
config.watermark_config = WatermarkConfig(
algorithm="dual_key_gumbel", key=42, alpha=alpha
)
if speculative:
config.speculative_config = SimpleNamespace(
method="mtp",
draft_sample_method="probabilistic",
rejection_sample_method="standard",
parallel_drafting=False,
)
with caplog_vllm.at_level(logging.WARNING):
config._check_watermarking_unsupported()
assert "is not used" not in caplog_vllm.text
def test_dual_key_gumbel_requires_probabilistic_drafting():
config = _watermarked_vllm_config()
config.watermark_config = WatermarkConfig(algorithm="dual_key_gumbel", key=42)
config.speculative_config = SimpleNamespace(
method="mtp",
draft_sample_method="greedy",
rejection_sample_method="standard",
parallel_drafting=False,
)
with pytest.raises(ValueError, match="draft_sample_method='probabilistic'"):
config._check_watermarking_unsupported()
@pytest.mark.parametrize("method", ["eagle", "eagle3", "mtp"])
def test_dual_key_gumbel_supports_probabilistic_speculative_decoding(method):
config = _watermarked_vllm_config()
config.watermark_config = WatermarkConfig(algorithm="dual_key_gumbel", key=42)
config.speculative_config = SimpleNamespace(
method=method,
draft_sample_method="probabilistic",
rejection_sample_method="standard",
parallel_drafting=False,
)
config._check_watermarking_unsupported()
def test_dual_key_gumbel_supports_dspark():
config = _watermarked_vllm_config()
config.watermark_config = WatermarkConfig(algorithm="dual_key_gumbel", key=42)
config.speculative_config = SimpleNamespace(
method="dspark",
draft_sample_method="probabilistic",
rejection_sample_method="standard",
parallel_drafting=True,
)
config._check_watermarking_unsupported()
def test_dual_key_gumbel_rejects_non_autoregressive_speculation():
config = _watermarked_vllm_config()
config.watermark_config = WatermarkConfig(algorithm="dual_key_gumbel", key=42)
config.speculative_config = SimpleNamespace(
method="ngram",
draft_sample_method="probabilistic",
rejection_sample_method="standard",
parallel_drafting=False,
)
with pytest.raises(ValueError, match="autoregressive model-based"):
config._check_watermarking_unsupported()
@pytest.mark.parametrize(
("overrides", "match"),
[
({"rejection_sample_method": "synthetic"}, "rejection_sample_method"),
({"rejection_sample_method": "block"}, "rejection_sample_method"),
({"parallel_drafting": True}, "Parallel speculative drafting"),
],
)
def test_dual_key_gumbel_rejects_incompatible_speculative_modes(overrides, match):
config = _watermarked_vllm_config()
config.watermark_config = WatermarkConfig(algorithm="dual_key_gumbel", key=42)
values = {
"method": "mtp",
"draft_sample_method": "probabilistic",
"rejection_sample_method": "standard",
"parallel_drafting": False,
**overrides,
}
config.speculative_config = SimpleNamespace(**values)
with pytest.raises(ValueError, match=match):
config._check_watermarking_unsupported()
@@ -2882,6 +3051,12 @@ def test_watermark_key_must_fit_in_64_bits():
WatermarkConfig(key=2**64)
@pytest.mark.parametrize("alpha", [-0.1, 1.1])
def test_watermark_alpha_must_be_a_probability(alpha):
with pytest.raises(ValidationError):
WatermarkConfig(key=42, algorithm="dual_key_gumbel", alpha=alpha)
def test_unknown_watermark_prf_is_rejected():
with pytest.raises(ValidationError):
pydantic.TypeAdapter(WatermarkConfig).validate_python(
@@ -6,7 +6,10 @@ import math
import pytest
import torch
from vllm.v1.watermarking import GumbelWatermarker
from vllm.v1.watermarking.spec_decode import watermarked_rejection_sample
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
from vllm.v1.worker.gpu.sample.watermark import philox_gumbel_sample
from vllm.v1.worker.gpu.spec_decode.rejection_sampler_utils import (
rejection_sample,
)
@@ -139,6 +142,93 @@ def _assert_distribution_match(
)
def test_watermarked_bonus_uses_target_key():
target_logits = torch.randn(16, device="cuda")
inputs = _build_rejection_sample_inputs(
target_logits,
target_logits,
num_speculative_steps=1,
temperature=1.0,
num_trials=1,
)
draft_token = inputs["draft_sampled"][1]
contexts = torch.tensor(
[[10, 11], [11, draft_token.item()]], dtype=torch.int64, device="cuda"
)
sampled, num_sampled = watermarked_rejection_sample(
**inputs,
num_speculative_steps=1,
contexts=contexts,
watermarking=torch.tensor([True], device="cuda"),
watermarker=GumbelWatermarker(key=42, context_width=2),
)
expected = GumbelWatermarker(key=42, context_width=2).sample(
target_logits.unsqueeze(0), contexts[1:], lambda _: None
)
assert num_sampled.item() == 2
assert sampled[0, 1] == expected.token_ids[0]
def test_watermarked_recovery_uses_target_key():
target_logits = torch.tensor([float("-inf"), 0.0, 1.0, 2.0], device="cuda")
draft_logits = torch.tensor([100.0, -100.0, -100.0, -100.0], device="cuda")
inputs = _build_rejection_sample_inputs(
target_logits,
draft_logits,
num_speculative_steps=1,
temperature=1.0,
num_trials=1,
)
inputs["draft_sampled"][1] = 0
contexts = torch.tensor([[10, 11], [11, 0]], dtype=torch.int64, device="cuda")
sampled, num_sampled = watermarked_rejection_sample(
**inputs,
num_speculative_steps=1,
contexts=contexts,
watermarking=torch.tensor([True], device="cuda"),
watermarker=GumbelWatermarker(key=42, context_width=2),
)
expected = GumbelWatermarker(key=42, context_width=2).sample(
target_logits.unsqueeze(0), contexts[:1], lambda _: None
)
assert num_sampled.item() == 1
assert sampled[0, 0] == expected.token_ids[0]
def test_watermarked_recovery_supports_smaller_draft_vocabulary():
target_logits = torch.tensor([float("-inf"), 0.0, 1.0, 2.0, 100.0], device="cuda")
draft_logits = torch.tensor([100.0, -100.0, -100.0, -100.0], device="cuda")
inputs = _build_rejection_sample_inputs(
target_logits[:4],
draft_logits,
num_speculative_steps=1,
temperature=1.0,
num_trials=1,
)
inputs["target_logits"] = target_logits.unsqueeze(0).expand(2, -1).contiguous()
inputs["draft_sampled"][1] = 0
contexts = torch.tensor([[10, 11], [11, 0]], dtype=torch.int64, device="cuda")
watermarker = GumbelWatermarker(key=42, context_width=2)
sampled, num_sampled = watermarked_rejection_sample(
**inputs,
num_speculative_steps=1,
contexts=contexts,
watermarking=torch.tensor([True], device="cuda"),
watermarker=watermarker,
)
expected = watermarker.sample(
target_logits[:4].unsqueeze(0), contexts[:1], lambda _: None
)
assert num_sampled.item() == 1
assert sampled[0, 0] == expected.token_ids[0]
@pytest.mark.parametrize(
"num_speculative_steps,temperature",
[
@@ -747,3 +837,192 @@ def test_chunked_requests_match_full_batch(has_draft_logits: bool):
steps = torch.arange(num_speculative_steps + 1, device=device)
valid = steps.unsqueeze(0) < num_sampled.unsqueeze(1)
assert torch.equal(chunked_sampled[valid], sampled[valid])
def _residual_recovery_logits(
inputs: dict,
num_sampled: torch.Tensor,
vocab_size: int,
) -> tuple[torch.Tensor, torch.Tensor]:
target_logits = inputs["target_logits"]
draft_logits = inputs["draft_logits"]
cu_num_logits = inputs["cu_num_logits"]
resample_rows = cu_num_logits[:-1].to(torch.int64) + num_sampled.to(torch.int64) - 1
req_state_indices = inputs["expanded_idx_mapping"][resample_rows].to(torch.int64)
temperatures = inputs["temperature"][req_state_indices]
is_bonus = resample_rows == cu_num_logits[1:] - 1
next_rows = (resample_rows + 1).clamp_max(inputs["draft_sampled"].shape[0] - 1)
needs_residual = ~is_bonus & (inputs["draft_sampled"][next_rows] >= 0)
target_rows = target_logits[resample_rows][:, :vocab_size].float()
target_rows = torch.where(target_rows.isnan(), float("-inf"), target_rows)
if draft_logits is None:
rejected = inputs["draft_sampled"][next_rows].clamp_min(0)
one_hot = torch.arange(vocab_size, device=target_rows.device).unsqueeze(
0
) == rejected.unsqueeze(-1)
residual_logits = torch.where(one_hot, float("-inf"), target_rows)
else:
draft_steps = (num_sampled.to(torch.int64) - 1).clamp_max(
draft_logits.shape[1] - 1
)
draft_rows = draft_logits[req_state_indices, draft_steps][
:, :vocab_size
].float()
draft_rows = torch.where(draft_rows.isnan(), float("-inf"), draft_rows)
draft_rows = draft_rows / temperatures.unsqueeze(-1)
target_log_probs = torch.log_softmax(target_rows, dim=-1)
draft_log_probs = torch.log_softmax(draft_rows, dim=-1)
ratio = torch.exp(draft_log_probs - target_log_probs)
residual_logits = torch.where(
ratio < 1,
target_log_probs + torch.log1p(-ratio.clamp_max(1)),
float("-inf"),
)
recovery = torch.where(needs_residual.unsqueeze(-1), residual_logits, target_rows)
return recovery, resample_rows
def _watermark_inputs(
num_trials: int,
num_speculative_steps: int,
context_width: int,
vocab_size: int,
one_hot_draft: bool,
temperature: float = 1.0,
) -> tuple[dict, torch.Tensor]:
device = "cuda"
target_logits_1d = torch.randn(vocab_size, device=device) / temperature
draft_logits_1d = torch.randn(vocab_size, device=device)
inputs = _build_rejection_sample_inputs(
target_logits_1d,
draft_logits_1d,
num_speculative_steps,
temperature=temperature,
num_trials=num_trials,
)
if one_hot_draft:
inputs["draft_logits"] = None
inputs["draft_sampled"].view(num_trials, num_speculative_steps + 1)[::5, 1] = -1
num_logits = inputs["target_logits"].shape[0]
contexts = torch.randint(
0, vocab_size, (num_logits, context_width), dtype=torch.int64, device=device
)
contexts[::7, 0] = -1
return inputs, contexts
@pytest.mark.parametrize("one_hot_draft", [False, True])
# Cover one full block and a partial multi-block vocabulary.
@pytest.mark.parametrize("vocab_size", [1024, 3500])
def test_watermarked_recovery_matches_philox_gumbel_sample(
one_hot_draft: bool, vocab_size: int
):
torch.manual_seed(7)
key = 0xDA39A3EE5E6B4B0D
context_width = 4
inputs, contexts = _watermark_inputs(
num_trials=128,
num_speculative_steps=2,
context_width=context_width,
vocab_size=vocab_size,
one_hot_draft=one_hot_draft,
)
watermarker = GumbelWatermarker(key=key, context_width=context_width)
sampled, num_sampled = watermarked_rejection_sample(
**inputs,
num_speculative_steps=2,
contexts=contexts,
watermarking=torch.ones(128, dtype=torch.bool, device="cuda"),
watermarker=watermarker,
)
recovery, resample_rows = _residual_recovery_logits(inputs, num_sampled, vocab_size)
expected = philox_gumbel_sample(recovery, contexts[resample_rows], key)
recovered = sampled[torch.arange(128, device="cuda"), num_sampled.long() - 1]
torch.testing.assert_close(recovered, expected, rtol=0, atol=0)
is_bonus = resample_rows == inputs["cu_num_logits"][1:] - 1
next_rows = (resample_rows + 1).clamp_max(inputs["draft_sampled"].shape[0] - 1)
is_placeholder = ~is_bonus & (inputs["draft_sampled"][next_rows] < 0)
assert int((~is_bonus & ~is_placeholder).sum()) > 0
assert int(is_placeholder.sum()) > 0
if not one_hot_draft:
# One-hot drafts almost never reach the bonus row.
assert int(is_bonus.sum()) > 0
def test_watermarked_recovery_leaves_disabled_rows_on_the_stock_draw():
torch.manual_seed(11)
num_trials = 96
vocab_size = 512
inputs, contexts = _watermark_inputs(
num_trials=num_trials,
num_speculative_steps=3,
context_width=2,
vocab_size=vocab_size,
one_hot_draft=False,
)
watermarking = torch.zeros(num_trials, dtype=torch.bool, device="cuda")
watermarking[::2] = True
inputs["temperature"][:8] = 0.0
stock_sampled, stock_num_sampled = rejection_sample(
**inputs, num_speculative_steps=3
)
sampled, num_sampled = watermarked_rejection_sample(
**inputs,
num_speculative_steps=3,
contexts=contexts,
watermarking=watermarking,
watermarker=GumbelWatermarker(key=99, context_width=2),
)
torch.testing.assert_close(num_sampled, stock_num_sampled, rtol=0, atol=0)
rows = torch.arange(num_trials, device="cuda")
positions = num_sampled.long() - 1
disabled = ~watermarking | (inputs["temperature"] == 0)
torch.testing.assert_close(
sampled[rows[disabled], positions[disabled]],
stock_sampled[rows[disabled], positions[disabled]],
rtol=0,
atol=0,
)
for req in range(num_trials):
n = int(num_sampled[req]) - 1
torch.testing.assert_close(
sampled[req, :n], stock_sampled[req, :n], rtol=0, atol=0
)
assert int(disabled.sum()) > 0 and int((~disabled).sum()) > 0
def test_watermarked_recovery_is_unchanged_by_fp64_gumbel():
torch.manual_seed(13)
inputs, contexts = _watermark_inputs(
num_trials=64,
num_speculative_steps=2,
context_width=4,
vocab_size=2048,
one_hot_draft=False,
)
watermarker = GumbelWatermarker(key=2**63 + 12345, context_width=4)
kwargs = dict(
num_speculative_steps=2,
contexts=contexts,
watermarking=torch.ones(64, dtype=torch.bool, device="cuda"),
watermarker=watermarker,
)
sampled32, num_sampled32 = watermarked_rejection_sample(**inputs, **kwargs)
sampled64, num_sampled64 = watermarked_rejection_sample(
**inputs, **kwargs, use_fp64=True
)
rows = torch.arange(64, device="cuda")
torch.testing.assert_close(num_sampled64, num_sampled32, rtol=0, atol=0)
torch.testing.assert_close(
sampled64[rows, num_sampled64.long() - 1],
sampled32[rows, num_sampled32.long() - 1],
rtol=0,
atol=0,
)
+125 -10
View File
@@ -5,8 +5,15 @@ import pytest
import torch
from vllm.platforms import current_platform
from vllm.v1.watermarking import GumbelWatermarkDetector, GumbelWatermarker
from vllm.v1.watermarking import (
DualKeyGumbelWatermarkDetector,
GumbelWatermarkDetector,
GumbelWatermarker,
derive_watermark_key,
)
from vllm.v1.watermarking.gumbel import _gamma_survival_integer_shape
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
from vllm.v1.worker.gpu.sample.watermark import philox_gumbel_sample
def test_gamma_survival_integer_shape():
@@ -34,6 +41,67 @@ def test_detector_deduplicates_context_even_when_target_differs():
assert detection.num_scored_tokens == 3
def test_dual_key_detector_scores_each_token_against_both_keys():
token_ids = [1, 2, 3, 4, 5]
key_a_detector = GumbelWatermarkDetector(
key=derive_watermark_key(42, b"key_a"),
context_width=1,
deduplicate_contexts=False,
)
key_b_detector = GumbelWatermarkDetector(
key=derive_watermark_key(42, b"key_b"),
context_width=1,
deduplicate_contexts=False,
)
detector = DualKeyGumbelWatermarkDetector(
key=42,
context_width=1,
deduplicate_contexts=False,
alpha=0.5,
)
key_a = key_a_detector.detect(token_ids)
key_b = key_b_detector.detect(token_ids)
dual = detector.detect(token_ids)
assert dual.score == pytest.approx((key_a.score + key_b.score) / 2)
assert dual.p_value == pytest.approx(
_gamma_survival_integer_shape(dual.score * 2, dual.num_scored_tokens * 2)
)
@pytest.mark.parametrize("alpha", [0.2, 0.7])
def test_dual_key_detector_weights_and_recalibrates_scores(alpha: float):
token_ids = [1, 2, 3, 4, 5]
key_a = GumbelWatermarkDetector(
key=derive_watermark_key(42, b"key_a"), deduplicate_contexts=False
).detect(token_ids)
key_b = GumbelWatermarkDetector(
key=derive_watermark_key(42, b"key_b"), deduplicate_contexts=False
).detect(token_ids)
dual = DualKeyGumbelWatermarkDetector(
key=42, deduplicate_contexts=False, alpha=alpha
).detect(token_ids)
variance = (1 - alpha) ** 2 + alpha**2
expected_p_value = torch.special.gammaincc(
torch.tensor(dual.num_scored_tokens / variance, dtype=torch.float64),
torch.tensor(dual.score / variance, dtype=torch.float64),
).item()
assert dual.score == pytest.approx((1 - alpha) * key_a.score + alpha * key_b.score)
assert dual.p_value == pytest.approx(expected_p_value)
def test_dual_key_detector_rejects_invalid_alpha():
with pytest.raises(ValueError, match="alpha must be between 0 and 1"):
DualKeyGumbelWatermarkDetector(key=42, alpha=1.1)
def test_dual_key_detector_default_alpha():
assert DualKeyGumbelWatermarkDetector(key=42).alpha == 0.2
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
@@ -45,10 +113,8 @@ def test_fused_watermarker_matches_cpu(key: int, context_width: int):
logits = torch.randn(32, 8193)
watermarker = GumbelWatermarker(key, context_width)
expected = watermarker.sample(logits, contexts, lambda values: None).token_ids
actual = watermarker.sample(
logits.cuda(), contexts.cuda(), lambda values: None
).token_ids.cpu()
expected = watermarker.sample(logits, contexts).token_ids
actual = watermarker.sample(logits.cuda(), contexts.cuda()).token_ids.cpu()
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
@@ -61,7 +127,7 @@ def test_fused_watermarker_handles_nan_logits():
logits = torch.full((2, 1025), float("nan"), device="cuda")
watermarker = GumbelWatermarker(key=42, context_width=4)
token_ids = watermarker.sample(logits, contexts, lambda values: None).token_ids
token_ids = watermarker.sample(logits, contexts).token_ids
assert torch.all((token_ids >= 0) & (token_ids < logits.shape[-1]))
@@ -76,9 +142,58 @@ def test_fused_watermarker_handles_noncontiguous_inputs():
logits = torch.randn(8, 2050, device="cuda")[:, ::2]
watermarker = GumbelWatermarker(key=42, context_width=4)
expected = watermarker.sample(
logits.contiguous(), contexts.contiguous(), lambda values: None
).token_ids
actual = watermarker.sample(logits, contexts, lambda values: None).token_ids
expected = watermarker.sample(logits.contiguous(), contexts.contiguous()).token_ids
actual = watermarker.sample(logits, contexts).token_ids
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
@pytest.mark.parametrize("use_fp64", [False, True])
def test_philox_gumbel_sample_skip_mask_matches_separate_samplers(use_fp64: bool):
torch.manual_seed(0)
logits = torch.randn(8, 8193, device="cuda")
context_storage = torch.randint(0, 248320, (8, 8), dtype=torch.int64, device="cuda")
contexts = context_storage[:, ::2]
repeated_mask = torch.tensor(
[False, False, False, True, False, False, True, False], device="cuda"
)
watermarking = torch.tensor(
[True, False, True, True, False, True, True, False], device="cuda"
)
req_indices = torch.tensor([3, 1, 7, 0, 4, 2, 6, 5], device="cuda")
temperatures = torch.ones(8, dtype=torch.float32, device="cuda")
temperatures[2] = 0
seeds = torch.arange(8, dtype=torch.int64, device="cuda") + 1000
positions = torch.arange(8, dtype=torch.int64, device="cuda") + 100
watermark_mask = (
watermarking[req_indices] & (temperatures[req_indices] != 0) & ~repeated_mask
)
watermarked = philox_gumbel_sample(logits, contexts, 42)
ordinary = gumbel_sample(
logits,
req_indices,
temperatures,
seeds,
positions,
apply_temperature=False,
is_drafting=False,
use_fp64=use_fp64,
)
expected = torch.where(watermark_mask, watermarked, ordinary)
actual = philox_gumbel_sample(
logits,
contexts,
42,
skip_mask=~watermark_mask,
expanded_idx_mapping=req_indices,
temperatures=temperatures,
seeds=seeds,
positions=positions,
use_fp64=use_fp64,
)
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
+889 -18
View File
@@ -9,44 +9,98 @@ import torch
from vllm import SamplingParams
from vllm.config.watermarking import WatermarkConfig
from vllm.v1.watermarking import create_watermarker
from vllm.platforms import current_platform
from vllm.v1.watermarking import (
DualKeyGumbelWatermarker,
SupportsSpeculativeDecoding,
create_watermarker,
derive_watermark_key,
)
from vllm.v1.watermarking.gpu_sampler import GPUWatermarkSampler
from vllm.v1.watermarking.watermarker import WatermarkSample
from vllm.v1.watermarking.gumbel import GumbelWatermarker
from vllm.v1.watermarking.spec_decode import (
DraftWatermarker,
_resolve_watermark_key,
create_speculative_draft_watermarker,
create_speculative_target_watermarker,
speculative_target_watermark_key,
)
from vllm.v1.watermarking.watermarker import Watermarker, WatermarkSample
from vllm.v1.worker.gpu.sample.sampler import Sampler
from vllm.v1.worker.gpu.sample.watermark import (
philox_gumbel_sample,
repeated_context_mask,
)
from vllm.v1.worker.gpu.spec_decode.dspark.speculator import DSparkSpeculator
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
@pytest.mark.parametrize("algorithm", ["gumbel"])
class StubWatermarker(Watermarker):
context_width = 1
def _sample_watermarked(self, logits, contexts):
return WatermarkSample(torch.tensor([7, 7]), logits + 10)
def _argmax_sampler(logits: torch.Tensor) -> torch.Tensor:
return logits.argmax(dim=-1)
@pytest.mark.parametrize("algorithm", ["gumbel", "dual_key_gumbel"])
def test_watermarker_contract(algorithm: str):
watermarker = create_watermarker(
WatermarkConfig(algorithm=algorithm, key=42, context_width=4)
)
logits = torch.zeros(2, 128)
contexts = torch.tensor([[1, 2, 3, 4], [4, 5, 6, 7]])
random_sample = lambda sample_logits: sample_logits.argmax(dim=-1)
first = watermarker.sample(logits, contexts, random_sample)
second = watermarker.sample(logits, contexts, random_sample)
first = watermarker.sample(logits, contexts, _argmax_sampler)
second = watermarker.sample(logits, contexts, _argmax_sampler)
assert first.token_ids.shape == (2,)
assert first.logits.shape == logits.shape
assert torch.equal(first.token_ids, second.token_ids)
def test_gumbel_config_warns_about_degenerate_generations(monkeypatch):
@pytest.mark.parametrize("algorithm", ["gumbel", "dual_key_gumbel"])
@pytest.mark.parametrize(
"config_overrides",
[
{"deduplicate_contexts": "none"},
{"deduplicate_contexts_max_history": 1023},
],
)
def test_gumbel_config_warns_when_context_deduplication_is_weak(
monkeypatch, algorithm, config_overrides
):
messages: list[str] = []
monkeypatch.setattr(
"vllm.config.watermarking.logger.warning_once",
lambda message, *, scope: messages.append(message),
)
WatermarkConfig(key=42)
WatermarkConfig(key=42, algorithm=algorithm)
WatermarkConfig(key=42, algorithm=algorithm, deduplicate_contexts_max_history=1024)
WatermarkConfig(key=42, algorithm=algorithm, deduplicate_contexts_max_history=None)
WatermarkConfig(key=42, algorithm=algorithm, **config_overrides)
assert messages == [
"Single-key Gumbel-max watermarking may increase the frequency of "
"degenerate generations, including repetition loops."
(
"Gumbel-max watermarking with context deduplication disabled "
"or limited to fewer than 1024 positions may increase the frequency of "
"degenerate generations, including repetition loops. Use "
"deduplicate_contexts='single_turn' or 'all' with "
"deduplicate_contexts_max_history at least 1024 or null to mitigate this."
)
]
def test_context_deduplication_history_can_be_unbounded():
config = WatermarkConfig(key=42, deduplicate_contexts_max_history=None)
assert config.deduplicate_contexts_max_history is None
def test_large_context_width_warns_but_is_allowed():
config = WatermarkConfig(key=42, context_width=17)
@@ -56,6 +110,158 @@ def test_large_context_width_warns_but_is_allowed():
assert watermarker.context_width == 17
def test_dual_key_watermarker_uses_domain_separated_keys():
config = WatermarkConfig(algorithm="dual_key_gumbel", key=42)
watermarker = create_watermarker(config)
assert isinstance(watermarker, SupportsSpeculativeDecoding)
draft = watermarker.draft_watermarker
target = watermarker.target_watermarker
assert isinstance(watermarker, DualKeyGumbelWatermarker)
assert watermarker.alpha == 0.1
assert draft.prf.key == derive_watermark_key(42, b"key_a")
assert target.prf.key == derive_watermark_key(42, b"key_b")
assert target.prf.key != draft.prf.key
def test_dual_key_watermarker_routes_tokens_with_alpha():
watermarker = DualKeyGumbelWatermarker(key=42, context_width=2, alpha=0.25)
logits = torch.zeros(2, 16)
contexts = torch.tensor([[1, 2], [3, 4]])
key_a = watermarker.draft_watermarker.sample(logits, contexts, lambda _: None)
key_b = watermarker.target_watermarker.sample(logits, contexts, lambda _: None)
def route(routing_logits):
torch.testing.assert_close(
routing_logits.softmax(dim=-1),
torch.tensor([[0.75, 0.25], [0.75, 0.25]]),
)
return torch.tensor([0, 1])
sampled = watermarker.sample(logits, contexts, route)
assert torch.equal(
sampled.token_ids, torch.stack([key_a.token_ids[0], key_b.token_ids[1]])
)
def test_dual_key_routing_logits_are_cached():
alpha = 0.25
watermarker = DualKeyGumbelWatermarker(key=42, context_width=2, alpha=alpha)
logits = torch.zeros(2, 16)
contexts = torch.tensor([[1, 2], [3, 4]])
recorded: list[torch.Tensor] = []
def route(routing_logits):
recorded.append(routing_logits)
return torch.tensor([0, 1])
watermarker.sample(logits, contexts, route)
watermarker.sample(logits, contexts, route)
expected = torch.tensor([1 - alpha, alpha], dtype=torch.float32).log()
expected = expected.expand(2, -1)
assert len(recorded) == 2
assert all(torch.equal(seen, expected) for seen in recorded)
assert len(watermarker._routing_logits_cache) == 1
@pytest.mark.parametrize("alpha", [0.0, 0.25, 1.0])
def test_dual_key_watermarker_skips_masked_rows(alpha: float):
watermarker = DualKeyGumbelWatermarker(key=42, context_width=2, alpha=alpha)
torch.manual_seed(0)
logits = torch.randn(4, 32)
contexts = torch.tensor([[1, 2], [3, 4], [5, 6], [7, 8]])
skip_mask = torch.tensor([True, False, True, False])
unmasked = watermarker.sample(logits, contexts, _argmax_sampler)
masked = watermarker.sample(logits, contexts, _argmax_sampler, skip_mask)
ordinary = _argmax_sampler(logits)
assert torch.equal(masked.token_ids[skip_mask], ordinary[skip_mask])
assert torch.equal(masked.token_ids[~skip_mask], unmasked.token_ids[~skip_mask])
assert not torch.equal(unmasked.token_ids[skip_mask], ordinary[skip_mask])
def test_dual_key_watermarker_routing_requires_a_random_sampler():
watermarker = DualKeyGumbelWatermarker(key=42, context_width=2, alpha=0.25)
with pytest.raises(ValueError, match="random sampler"):
watermarker.sample(torch.zeros(2, 16), torch.tensor([[1, 2], [3, 4]]))
def test_speculative_decoding_uses_fixed_dual_key_roles():
watermarker = create_watermarker(
WatermarkConfig(algorithm="dual_key_gumbel", key=42, alpha=0.25)
)
target = create_speculative_target_watermarker(watermarker)
draft = create_speculative_draft_watermarker(
watermarker,
max_num_reqs=1,
device=torch.device("cpu"),
allow_target_only=False,
)
assert target.prf.key == derive_watermark_key(42, b"key_b")
assert draft is not None
assert draft.watermarker.prf.key == derive_watermark_key(42, b"key_a")
def test_recovery_key_rejects_the_unsplit_dual_key_watermarker():
watermarker = create_watermarker(
WatermarkConfig(algorithm="dual_key_gumbel", key=42, alpha=0.25)
)
with pytest.raises(ValueError, match="keys the target role separately"):
_resolve_watermark_key(watermarker)
target = create_speculative_target_watermarker(watermarker)
assert _resolve_watermark_key(target) == derive_watermark_key(42, b"key_b")
@pytest.mark.parametrize("algorithm", ["gumbel", "dual_key_gumbel"])
def test_config_key_resolution_matches_the_model_runner(algorithm: str):
config = WatermarkConfig(algorithm=algorithm, key=42, alpha=0.25)
runtime_key = _resolve_watermark_key(
create_speculative_target_watermarker(create_watermarker(config))
)
assert speculative_target_watermark_key(config) == runtime_key
def test_config_key_resolution_returns_none_when_watermarking_is_disabled():
assert speculative_target_watermark_key(None) is None
def test_target_only_speculative_watermarking_skips_draft_watermarker():
watermarker = create_watermarker(WatermarkConfig(algorithm="gumbel", key=42))
assert not isinstance(watermarker, SupportsSpeculativeDecoding)
with pytest.raises(ValueError, match="does not support speculative decoding"):
create_speculative_draft_watermarker(
watermarker,
max_num_reqs=1,
device=torch.device("cpu"),
allow_target_only=False,
)
assert (
create_speculative_draft_watermarker(
watermarker,
max_num_reqs=1,
device=torch.device("cpu"),
allow_target_only=True,
)
is None
)
def test_dual_key_derivation_is_stable():
assert derive_watermark_key(32, b"key_a") == 16368605726115524094
assert derive_watermark_key(32, b"key_b") == 4799302812959726346
def test_sampling_params_can_disable_watermarking():
assert SamplingParams().watermarking
assert not SamplingParams.from_optional(watermarking=False).watermarking
@@ -83,14 +289,9 @@ def test_gpu_sampler_warns_when_watermarking_is_enabled_for_greedy(monkeypatch):
def test_gpu_sampler_respects_mixed_request_watermarking(monkeypatch):
class StubWatermarker:
context_width = 1
def sample(self, logits, contexts, random_sample):
return WatermarkSample(torch.tensor([7, 7]), logits + 10)
sampler = object.__new__(GPUWatermarkSampler)
sampler.watermarker = StubWatermarker()
sampler.deduplicate_contexts = "single_turn"
sampler.watermarking = SimpleNamespace(
np=np.array([True, False]), gpu=torch.tensor([True, False])
)
@@ -102,8 +303,11 @@ def test_gpu_sampler_respects_mixed_request_watermarking(monkeypatch):
sampler._get_contexts = lambda expanded_idx_mapping: torch.zeros(
2, 1, dtype=torch.int64
)
sampler._get_repeated_contexts = lambda expanded_idx_mapping, contexts: torch.zeros(
2, dtype=torch.bool
)
monkeypatch.setattr(
"vllm.v1.watermarking.gpu_sampler.gumbel_sample",
"vllm.v1.watermarking.watermarker.gumbel_sample",
lambda *args, **kwargs: torch.tensor([3, 4]),
)
logits = torch.zeros(2, 8)
@@ -123,11 +327,524 @@ def test_gpu_sampler_respects_mixed_request_watermarking(monkeypatch):
assert torch.equal(output_logits[1], logits[1])
def test_gpu_sampler_skips_watermarking_for_repeated_contexts(monkeypatch):
sampler = object.__new__(GPUWatermarkSampler)
sampler.watermarker = StubWatermarker()
sampler.deduplicate_contexts = "single_turn"
sampler.watermarking = SimpleNamespace(
np=np.array([True, True]), gpu=torch.tensor([True, True])
)
sampler.sampling_states = SimpleNamespace(
temperature=SimpleNamespace(np=np.ones(2), gpu=torch.ones(2)),
seeds=SimpleNamespace(gpu=torch.zeros(2, dtype=torch.int64)),
)
sampler.use_fp64_gumbel = False
sampler._get_contexts = lambda expanded_idx_mapping: torch.zeros(
2, 1, dtype=torch.int64
)
sampler._get_repeated_contexts = lambda expanded_idx_mapping, contexts: (
torch.tensor([True, False])
)
monkeypatch.setattr(
"vllm.v1.watermarking.watermarker.gumbel_sample",
lambda *args, **kwargs: torch.tensor([3, 4]),
)
logits = torch.zeros(2, 8)
sampled, output_logits = sampler._sample_random(
logits,
torch.tensor([0, 1]),
np.array([0, 1]),
torch.zeros(2, dtype=torch.int64),
None,
None,
False,
)
assert torch.equal(sampled, torch.tensor([3, 7]))
assert torch.equal(output_logits[0], logits[0])
assert torch.equal(output_logits[1], torch.full((8,), 10.0))
def test_gpu_sampler_can_disable_context_deduplication(monkeypatch):
sampler = object.__new__(GPUWatermarkSampler)
sampler.watermarker = StubWatermarker()
sampler.deduplicate_contexts = "none"
sampler.watermarking = SimpleNamespace(
np=np.array([True, True]), gpu=torch.tensor([True, True])
)
sampler.sampling_states = SimpleNamespace(
temperature=SimpleNamespace(np=np.ones(2), gpu=torch.ones(2)),
seeds=SimpleNamespace(gpu=torch.zeros(2, dtype=torch.int64)),
)
sampler.use_fp64_gumbel = False
sampler._get_contexts = lambda expanded_idx_mapping: torch.zeros(
2, 1, dtype=torch.int64
)
sampler._get_repeated_contexts = lambda *args: pytest.fail(
"context deduplication should not run"
)
logits = torch.zeros(2, 8)
sampled, output_logits = sampler._sample_random(
logits,
torch.tensor([0, 1]),
np.array([0, 1]),
torch.zeros(2, dtype=torch.int64),
None,
None,
False,
)
assert torch.equal(sampled, torch.tensor([7, 7]))
assert torch.equal(output_logits, torch.full((2, 8), 10.0))
def test_repeated_context_mask_ignores_prompt_tokens():
all_token_ids = torch.tensor(
[
[8, 9, 1, 2, 1, 2],
[3, 4, 1, 2, 3, 4],
[0, 0, 0, 0, 0, 0],
],
dtype=torch.int32,
)
req_indices = torch.tensor([0, 1, -1])
prompt_lens = torch.tensor([2, 2, 0])
total_lens = torch.tensor([6, 6, 0])
contexts = torch.tensor([[1, 2], [3, 4], [-1, -1]])
repeated = repeated_context_mask(
all_token_ids,
req_indices,
prompt_lens,
total_lens,
contexts,
)
assert torch.equal(repeated, torch.tensor([True, False, False]))
def test_repeated_context_mask_can_include_prompt_tokens():
all_token_ids = torch.tensor([[1, 2, 3, 4, 5, 6, 1, 2, 3, 4]])
req_indices = torch.tensor([0])
prompt_lens = torch.tensor([4])
total_lens = torch.tensor([10])
contexts = torch.tensor([[1, 2, 3, 4]])
single_turn = repeated_context_mask(
all_token_ids, req_indices, prompt_lens, total_lens, contexts
)
all_history = repeated_context_mask(
all_token_ids,
req_indices,
prompt_lens,
total_lens,
contexts,
include_prompt=True,
)
assert not single_turn.item()
assert all_history.item()
def test_repeated_context_mask_can_skip_partial_contexts():
all_token_ids = torch.tensor([[10, 11, 12, 13, 1, 2, 3, 4]], dtype=torch.int32)
req_indices = torch.tensor([0])
prompt_lens = torch.tensor([4])
contexts = torch.tensor([[-1, -1, -1, 1]])
partial = repeated_context_mask(
all_token_ids,
req_indices,
prompt_lens,
torch.tensor([5]),
contexts,
include_prompt=True,
skip_partial_context=True,
)
complete = repeated_context_mask(
all_token_ids,
req_indices,
prompt_lens,
torch.tensor([8]),
torch.tensor([[1, 2, 3, 4]]),
include_prompt=True,
skip_partial_context=True,
)
assert partial.item()
assert not complete.item()
def test_gpu_sampler_contexts_are_completion_local_for_all_scopes():
sampler = object.__new__(GPUWatermarkSampler)
sampler.watermarker = SimpleNamespace(context_width=4)
sampler.req_states = SimpleNamespace(
all_token_ids=SimpleNamespace(gpu=torch.tensor([[10, 11, 12, 13, 1, 2]])),
prompt_len=SimpleNamespace(gpu=torch.tensor([4])),
total_len=SimpleNamespace(gpu=torch.tensor([6])),
)
request_indices = torch.tensor([0])
sampler.deduplicate_contexts = "single_turn"
single_turn = sampler._get_contexts(request_indices)
sampler.deduplicate_contexts = "all"
all_history = sampler._get_contexts(request_indices)
assert torch.equal(single_turn, torch.tensor([[-1, -1, 1, 2]]))
assert torch.equal(all_history, single_turn)
def test_gpu_sampler_all_history_skips_partial_context():
sampler = object.__new__(GPUWatermarkSampler)
sampler.watermarker = SimpleNamespace(context_width=4)
sampler.req_states = SimpleNamespace(
all_token_ids=SimpleNamespace(gpu=torch.tensor([[10, 11, 12, 13, 1, 2]])),
prompt_len=SimpleNamespace(gpu=torch.tensor([4])),
total_len=SimpleNamespace(gpu=torch.tensor([6])),
)
sampler.deduplicate_contexts_max_history = None
request_indices = torch.tensor([0])
contexts = sampler._get_contexts(request_indices)
sampler.deduplicate_contexts = "single_turn"
single_turn = sampler._get_repeated_contexts(request_indices, contexts)
sampler.deduplicate_contexts = "all"
all_history = sampler._get_repeated_contexts(request_indices, contexts)
assert not single_turn.item()
assert all_history.item()
def test_repeated_context_mask_respects_max_history():
all_token_ids = torch.tensor([[1, 2, 3, 4, 5, 6, 1, 2, 3, 4]])
req_indices = torch.tensor([0])
prompt_lens = torch.tensor([0])
total_lens = torch.tensor([10])
contexts = torch.tensor([[1, 2, 3, 4]])
full_history = repeated_context_mask(
all_token_ids, req_indices, prompt_lens, total_lens, contexts
)
last_six = repeated_context_mask(
all_token_ids, req_indices, prompt_lens, total_lens, contexts, max_history=6
)
last_five = repeated_context_mask(
all_token_ids, req_indices, prompt_lens, total_lens, contexts, max_history=5
)
assert full_history.item()
assert last_six.item()
assert not last_five.item()
@pytest.mark.parametrize("max_history", [0, -1])
def test_repeated_context_mask_rejects_non_positive_max_history(max_history: int):
with pytest.raises(ValueError, match="max_history must be positive or None"):
repeated_context_mask(
torch.tensor([[1, 2, 3, 4]], dtype=torch.int32),
torch.tensor([0], dtype=torch.int32),
torch.tensor([0], dtype=torch.int32),
torch.tensor([4], dtype=torch.int32),
torch.tensor([[1, 2, 3, 4]], dtype=torch.int32),
max_history=max_history,
)
def test_gpu_sampler_wires_request_history_scope():
sampler = object.__new__(GPUWatermarkSampler)
sampler.req_states = SimpleNamespace(
all_token_ids=SimpleNamespace(
gpu=torch.tensor([[1, 2, 3, 4, 5, 6, 1, 2, 3, 4]])
),
prompt_len=SimpleNamespace(gpu=torch.tensor([4])),
total_len=SimpleNamespace(gpu=torch.tensor([10])),
)
sampler.deduplicate_contexts_max_history = None
request_indices = torch.tensor([0])
contexts = torch.tensor([[1, 2, 3, 4]])
sampler.deduplicate_contexts = "single_turn"
single_turn = sampler._get_repeated_contexts(request_indices, contexts)
sampler.deduplicate_contexts = "all"
all_history = sampler._get_repeated_contexts(request_indices, contexts)
assert not single_turn.item()
assert all_history.item()
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
@pytest.mark.parametrize("max_history", [5, 6])
@pytest.mark.parametrize("include_prompt", [False, True])
def test_repeated_context_mask_max_history_accelerator_parity(
max_history: int, include_prompt: bool
):
inputs = (
torch.tensor([[1, 2, 3, 4, 5, 6, 1, 2, 3, 4]]),
torch.tensor([0]),
torch.tensor([0]),
torch.tensor([10]),
torch.tensor([[1, 2, 3, 4]]),
)
expected = repeated_context_mask(
*inputs, max_history=max_history, include_prompt=include_prompt
)
actual = repeated_context_mask(
*(value.cuda() for value in inputs),
max_history=max_history,
include_prompt=include_prompt,
).cpu()
assert torch.equal(actual, expected)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
def test_repeated_context_mask_partial_context_accelerator_parity():
inputs = (
torch.tensor([[10, 11, 12, 13, 1]], dtype=torch.int32),
torch.tensor([0], dtype=torch.int32),
torch.tensor([4], dtype=torch.int32),
torch.tensor([5], dtype=torch.int32),
torch.tensor([[-1, -1, -1, 1]], dtype=torch.int64),
)
expected = repeated_context_mask(
*inputs, include_prompt=True, skip_partial_context=True
)
actual = repeated_context_mask(
*(value.cuda() for value in inputs),
include_prompt=True,
skip_partial_context=True,
).cpu()
assert torch.equal(actual, expected)
def _unaligned_max_history_inputs() -> tuple[torch.Tensor, ...]:
"""One row whose only repeated context ends at position 680.
680 is not a multiple of the kernel's 512-position block, so the scan window
selected by ``max_history`` starts in the middle of a block.
"""
history = list(range(1_000, 2_200))
history[676:680] = [1, 2, 3, 4]
return (
torch.tensor([history], dtype=torch.int32),
torch.tensor([0], dtype=torch.int32),
torch.tensor([0], dtype=torch.int32),
torch.tensor([len(history)], dtype=torch.int32),
torch.tensor([[1, 2, 3, 4]], dtype=torch.int32),
)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
@pytest.mark.parametrize(
"max_history,expected", [(1200 - 680, True), (1200 - 680 - 1, False)]
)
def test_repeated_context_mask_unaligned_max_history_accelerator_parity(
max_history: int, expected: bool
):
inputs = _unaligned_max_history_inputs()
reference = repeated_context_mask(*inputs, max_history=max_history)
actual = repeated_context_mask(
*(value.cuda() for value in inputs), max_history=max_history
).cpu()
assert torch.equal(reference, torch.tensor([expected]))
assert torch.equal(actual, reference)
def _repeated_context_inputs(
context_width: int, device: str = "cpu"
) -> tuple[torch.Tensor, ...]:
"""Build repeated, unique, and padded request rows for mask tests."""
prompt = [101, 102, 103, 104]
repeated_output = [*range(1, context_width + 1)] * 2
unique_output = [*range(1, context_width + 2)]
max_len = len(prompt) + len(repeated_output)
rows = [
prompt + repeated_output,
prompt + unique_output,
[],
]
all_token_ids = torch.zeros((3, max_len), dtype=torch.int32, device=device)
for row, token_ids in enumerate(rows):
all_token_ids[row, : len(token_ids)] = torch.tensor(
token_ids, dtype=torch.int32, device=device
)
return (
all_token_ids,
torch.tensor([0, 1, -1], dtype=torch.int32, device=device),
torch.tensor([len(prompt), len(prompt), 0], dtype=torch.int32, device=device),
torch.tensor([len(rows[0]), len(rows[1]), 0], dtype=torch.int32, device=device),
torch.tensor(
[
repeated_output[-context_width:],
unique_output[-context_width:],
[-1] * context_width,
],
dtype=torch.int64,
device=device,
),
)
@pytest.mark.parametrize("context_width", [1, 3, 4, 16, 17])
def test_repeated_context_mask(context_width: int):
repeated = repeated_context_mask(*_repeated_context_inputs(context_width))
assert torch.equal(repeated, torch.tensor([True, False, False]))
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
@pytest.mark.parametrize("context_width", [1, 3, 4, 16, 17])
def test_repeated_context_mask_accelerator_parity(context_width: int):
cpu_inputs = _repeated_context_inputs(context_width)
accelerator_inputs = list(_repeated_context_inputs(context_width, "cuda"))
contexts = accelerator_inputs[-1]
storage = torch.empty(
contexts.shape[0], contexts.shape[1] * 2, dtype=contexts.dtype, device="cuda"
)
storage[:, ::2] = contexts
accelerator_inputs[-1] = storage[:, ::2]
expected = repeated_context_mask(*cpu_inputs)
actual = repeated_context_mask(*accelerator_inputs).cpu()
assert torch.equal(actual, expected)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
def test_repeated_context_mask_scans_multiple_blocks():
context = [1, 2, 3, 4]
repeated_output = [*range(10_000, 11_030), *context, 99, *context]
unique_output = list(range(20_000, 21_039))
prompt = [101, 102, 103, 104]
rows = [prompt + repeated_output, prompt + unique_output]
all_token_ids = torch.zeros((2, len(rows[0])), dtype=torch.int32)
for row, token_ids in enumerate(rows):
all_token_ids[row, : len(token_ids)] = torch.tensor(token_ids)
inputs = (
all_token_ids,
torch.tensor([0, 1], dtype=torch.int32),
torch.tensor([len(prompt), len(prompt)], dtype=torch.int32),
torch.tensor([len(rows[0]), len(rows[1])], dtype=torch.int32),
torch.tensor([context, unique_output[-4:]], dtype=torch.int64),
)
expected = repeated_context_mask(*inputs)
actual = repeated_context_mask(*(value.cuda() for value in inputs)).cpu()
assert torch.equal(expected, torch.tensor([True, False]))
assert torch.equal(actual, expected)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
def test_repeated_context_mask_last_request_at_capacity():
max_model_len = 1024
all_token_ids = torch.arange(
2 * max_model_len, dtype=torch.int32, device="cuda"
).reshape(2, max_model_len)
inputs = (
all_token_ids,
torch.tensor([1], dtype=torch.int32, device="cuda"),
torch.tensor([0, 0], dtype=torch.int32, device="cuda"),
torch.tensor([max_model_len, max_model_len], dtype=torch.int32, device="cuda"),
all_token_ids[1, -4:].to(torch.int64).unsqueeze(0),
)
repeated = repeated_context_mask(*inputs)
assert not repeated.item()
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
def test_gpu_sampler_uses_fused_gumbel_for_repeated_contexts():
device = torch.device("cuda")
logits = torch.tensor([[0.0, 1.0, 2.0, 3.0], [3.0, 2.0, 1.0, 0.0]], device=device)
all_token_ids = torch.tensor(
[[101, 102, 1, 2, 1, 2], [101, 102, 3, 4, 5, 6]],
dtype=torch.int32,
device=device,
)
request_indices = torch.tensor([0, 1], dtype=torch.int64, device=device)
request_indices_np = np.array([0, 1])
temperatures = torch.ones(2, device=device)
seeds = torch.tensor([11, 22], dtype=torch.int64, device=device)
positions = torch.tensor([6, 6], dtype=torch.int64, device=device)
sampler = object.__new__(GPUWatermarkSampler)
sampler.watermarker = GumbelWatermarker(key=42, context_width=1)
sampler.deduplicate_contexts = "single_turn"
sampler.deduplicate_contexts_max_history = None
sampler.watermarking = SimpleNamespace(
np=np.ones(2, dtype=bool), gpu=torch.ones(2, dtype=torch.bool, device=device)
)
sampler.sampling_states = SimpleNamespace(
temperature=SimpleNamespace(np=np.ones(2), gpu=temperatures),
seeds=SimpleNamespace(gpu=seeds),
)
sampler.req_states = SimpleNamespace(
all_token_ids=SimpleNamespace(gpu=all_token_ids),
prompt_len=SimpleNamespace(
gpu=torch.tensor([2, 2], dtype=torch.int32, device=device)
),
total_len=SimpleNamespace(
gpu=torch.tensor([6, 6], dtype=torch.int32, device=device)
),
)
sampler.use_fp64_gumbel = False
contexts = sampler._get_contexts(request_indices)
repeated = sampler._get_repeated_contexts(request_indices, contexts)
expected = philox_gumbel_sample(
logits,
contexts,
42,
skip_mask=repeated,
expanded_idx_mapping=request_indices,
temperatures=temperatures,
seeds=seeds,
positions=positions,
)
actual, output_logits = sampler._sample_random(
logits,
request_indices,
request_indices_np,
positions,
None,
None,
False,
)
assert torch.equal(repeated, torch.tensor([True, False], device=device))
assert torch.equal(actual, expected)
assert output_logits is logits
def test_gpu_sampler_skips_watermarking_for_greedy_batch(monkeypatch):
class StubWatermarker:
context_width = 1
def sample(self, logits, contexts, random_sample):
def sample(self, logits, contexts, random_sampler=None, skip_mask=None):
raise AssertionError("watermarker should not run for greedy requests")
sampler = object.__new__(GPUWatermarkSampler)
@@ -155,3 +872,157 @@ def test_gpu_sampler_skips_watermarking_for_greedy_batch(monkeypatch):
)
assert actual is expected
def test_gpu_sampler_builds_speculative_contexts_from_drafts():
sampler = object.__new__(GPUWatermarkSampler)
sampler.watermarker = SimpleNamespace(context_width=2)
sampler.req_states = SimpleNamespace(
all_token_ids=SimpleNamespace(gpu=torch.tensor([[100, 101, 10, 11, 0, 0]])),
prompt_len=SimpleNamespace(gpu=torch.tensor([2])),
total_len=SimpleNamespace(gpu=torch.tensor([4])),
)
contexts = sampler._get_contexts(
torch.tensor([0, 0, 0]),
torch.tensor([0, 1, 2]),
torch.tensor([11, 20, 21]),
)
assert torch.equal(contexts, torch.tensor([[10, 11], [11, 20], [20, 21]]))
def test_gpu_sampler_builds_chunked_multi_request_speculative_contexts():
sampler = object.__new__(GPUWatermarkSampler)
sampler.watermarker = SimpleNamespace(context_width=2)
sampler.req_states = SimpleNamespace(
all_token_ids=SimpleNamespace(
gpu=torch.tensor(
[
[100, 101, 10, 11, 0, 0],
[200, 201, 30, 31, 0, 0],
[300, 301, 50, 51, 0, 0],
]
)
),
prompt_len=SimpleNamespace(gpu=torch.tensor([2, 2, 2])),
total_len=SimpleNamespace(gpu=torch.tensor([4, 4, 4])),
)
contexts = sampler._get_contexts(
torch.tensor([2, 2, 1, 1]),
torch.tensor([0, 1, 0, 1]),
torch.tensor([51, 60, 31, 40]),
)
assert torch.equal(
contexts,
torch.tensor([[50, 51], [51, 60], [30, 31], [31, 40]]),
)
def test_draft_sampler_uses_draft_key_and_advances_context(monkeypatch):
class StubSpeculator(DraftModelSpeculator):
def capture(self): ...
def init_cudagraph_manager(self, cudagraph_mode): ...
def load_draft_model(self, target_model, target_attn_layer_names): ...
def propose(self, *args, **kwargs): ...
class StubModel:
@staticmethod
def compute_logits(hidden_states):
return torch.zeros(hidden_states.shape[0], 8)
class StubWatermarker:
@staticmethod
def sample(logits, contexts, random_sampler=None):
assert random_sampler is None
return WatermarkSample(torch.tensor([7, 7]), logits)
speculator = object.__new__(StubSpeculator)
speculator.model = StubModel()
speculator.use_fp64_gumbel = False
draft_watermarker = object.__new__(DraftWatermarker)
draft_watermarker.watermarker = StubWatermarker()
draft_watermarker.contexts = torch.tensor([[1, 2], [3, 4]])
draft_watermarker.enabled = torch.tensor([True, False])
speculator.draft_watermarker = draft_watermarker
monkeypatch.setattr(
"vllm.v1.worker.gpu.spec_decode.speculator.gumbel_sample",
lambda *args, **kwargs: torch.tensor([3, 4]),
)
sampled = speculator.sample_draft(
hidden_states=torch.zeros(2, 4),
sample_src_positions=torch.zeros(2, dtype=torch.int64),
idx_mapping=torch.tensor([0, 1]),
temperature=torch.ones(2),
seeds=torch.zeros(2, dtype=torch.int64),
draft_step=torch.tensor(0),
draft_logits=torch.zeros(2, 1, 8),
)
assert torch.equal(sampled, torch.tensor([7, 4]))
assert torch.equal(draft_watermarker.contexts, torch.tensor([[2, 7], [4, 4]]))
def test_dspark_reduced_vocab_draft_sampler_applies_watermarking(monkeypatch):
speculator = object.__new__(DSparkSpeculator)
speculator.draft_logits = torch.zeros(2, 1, 8)
speculator._d2t_scatter_index = torch.tensor([1, 5])
speculator._draft_scatter_buf = torch.full((2, 8), float("-inf"))
speculator.temperature = torch.ones(2)
speculator.seeds = torch.zeros(2, dtype=torch.int64)
speculator._step_cols = torch.tensor([0])
speculator.use_fp64_gumbel = False
watermark_logits: list[torch.Tensor] = []
def sample(logits, sampled, idx_map, temperature):
watermark_logits.append(logits.clone())
return sampled + 1
speculator.draft_watermarker = SimpleNamespace(sample=sample)
monkeypatch.setattr(
"vllm.v1.worker.gpu.spec_decode.dspark.speculator.gumbel_sample",
lambda *args, **kwargs: torch.tensor([3, 4]),
)
sampled = speculator._sample_logits(
torch.tensor([[10.0, 20.0], [30.0, 40.0]]),
torch.tensor([0, 1]),
torch.tensor([1, 1]),
0,
)
assert torch.equal(sampled, torch.tensor([4, 5]))
assert torch.equal(
watermark_logits[0][:, [1, 5]], torch.tensor([[10, 20], [30, 40]])
)
assert torch.isneginf(watermark_logits[0][:, [0, 2, 3, 4, 6, 7]]).all()
def test_dspark_target_only_watermarking_leaves_drafts_unwatermarked(monkeypatch):
speculator = object.__new__(DSparkSpeculator)
speculator.draft_logits = torch.zeros(2, 1, 8)
speculator._d2t_scatter_index = None
speculator.temperature = torch.ones(2)
speculator.seeds = torch.zeros(2, dtype=torch.int64)
speculator._step_cols = torch.tensor([0])
speculator.use_fp64_gumbel = False
speculator.draft_watermarker = None
monkeypatch.setattr(
"vllm.v1.worker.gpu.spec_decode.dspark.speculator.gumbel_sample",
lambda *args, **kwargs: torch.tensor([3, 4]),
)
sampled = speculator._sample_logits(
torch.zeros(2, 8),
torch.tensor([0, 1]),
torch.tensor([1, 1]),
0,
)
assert torch.equal(sampled, torch.tensor([3, 4]))