[watermarking] Dual-key gumbel-max watermarking for speculative decoding support (#56122)
Signed-off-by: Raphael Rialland <raphael.rialland@mistral.ai> Signed-off-by: Simon Veitner <sveitner@redhat.com> Signed-off-by: Tomas Ruiz <tomas.ruiz.te@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Simon Veitner <sveitner@redhat.com> Co-authored-by: Tomas Ruiz <tomas.ruiz.te@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
Simon Veitner
Tomas Ruiz
parent
1ee4be4dbc
commit
7ee8a6dd01
@@ -53,18 +53,37 @@ def test_watermark_config_cli():
|
||||
"--model",
|
||||
"dummy",
|
||||
"--watermark-config",
|
||||
'{"algorithm":"gumbel","key":42,"prf":"philox"}',
|
||||
'{"algorithm":"dual_key_gumbel","key":42,"prf":"philox","alpha":0.25,'
|
||||
'"allow_target_only_watermarking":true}',
|
||||
]
|
||||
)
|
||||
|
||||
config = EngineArgs.from_cli_args(args).create_watermark_config()
|
||||
|
||||
assert config is not None
|
||||
assert config.algorithm == "gumbel"
|
||||
assert config.algorithm == "dual_key_gumbel"
|
||||
assert config.key == 42
|
||||
assert config.alpha == 0.25
|
||||
assert config.context_width == 4
|
||||
assert config.deduplicate_contexts == "single_turn"
|
||||
assert config.deduplicate_contexts_max_history == 8192
|
||||
assert config.prf == "philox"
|
||||
assert not config.supports_speculative_decoding
|
||||
assert config.allow_target_only_watermarking
|
||||
|
||||
args = parser.parse_args(
|
||||
[
|
||||
"--model",
|
||||
"dummy",
|
||||
"--watermark-config",
|
||||
'{"key":42,"deduplicate_contexts":"none",'
|
||||
'"deduplicate_contexts_max_history":32}',
|
||||
]
|
||||
)
|
||||
config = EngineArgs.from_cli_args(args).create_watermark_config()
|
||||
|
||||
assert config is not None
|
||||
assert config.deduplicate_contexts == "none"
|
||||
assert config.deduplicate_contexts_max_history == 32
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
model_name: "Qwen/Qwen3-4B-FP8"
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
max_tokens: 256
|
||||
use_chat_completions: false
|
||||
temperature: 1.0
|
||||
seed: 42
|
||||
|
||||
# Reference accuracy from the non-watermarked DSpark regression test.
|
||||
accuracy_threshold: 0.801
|
||||
tolerance: 0.02
|
||||
# Measured TP1 acceptance length: 4.115; allow a 0.2 regression.
|
||||
min_acceptance_length: 3.915
|
||||
|
||||
env:
|
||||
VLLM_USE_V2_MODEL_RUNNER: "1"
|
||||
server_args: >-
|
||||
--tensor-parallel-size 2
|
||||
--max-model-len 4096
|
||||
--gpu-memory-utilization 0.92
|
||||
--no-enable-prefix-caching
|
||||
--speculative-config.method dspark
|
||||
--speculative-config.model deepseek-ai/dspark_qwen3_4b_block7
|
||||
--speculative-config.num_speculative_tokens 7
|
||||
--speculative-config.draft_sample_method probabilistic
|
||||
--watermark-config.algorithm dual_key_gumbel
|
||||
--watermark-config.key 42
|
||||
--watermark-config.context_width 4
|
||||
--watermark-config.deduplicate_contexts none
|
||||
@@ -0,0 +1 @@
|
||||
Qwen3-4B-FP8-DSpark-watermark-TP2.yaml
|
||||
@@ -0,0 +1,121 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.watermarking import WatermarkConfig, derive_watermark_key
|
||||
from vllm.model_executor.warmup.spec_decode_rejection_warmup import (
|
||||
spec_decode_rejection_warmup,
|
||||
)
|
||||
|
||||
MASTER_KEY = 20250910
|
||||
|
||||
|
||||
def _worker(
|
||||
watermark_config: WatermarkConfig | None,
|
||||
draft_sample_method: str = "greedy",
|
||||
) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
device=torch.device("cpu"),
|
||||
vllm_config=SimpleNamespace(
|
||||
speculative_config=SimpleNamespace(
|
||||
num_speculative_tokens=2,
|
||||
rejection_sample_method="standard",
|
||||
draft_sample_method=draft_sample_method,
|
||||
),
|
||||
model_config=SimpleNamespace(
|
||||
get_vocab_size=lambda: 128, dtype=torch.bfloat16
|
||||
),
|
||||
watermark_config=watermark_config,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _capture_calls(monkeypatch: pytest.MonkeyPatch) -> list[dict]:
|
||||
calls: list[dict] = []
|
||||
|
||||
def fake_rejection_sample(**kwargs):
|
||||
calls.append(kwargs)
|
||||
return None, None
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.worker.gpu.spec_decode.rejection_sampler_utils.rejection_sample",
|
||||
fake_rejection_sample,
|
||||
)
|
||||
return calls
|
||||
|
||||
|
||||
def test_warmup_token_id_dtypes_match_the_runtime_buffers(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
calls = _capture_calls(monkeypatch)
|
||||
spec_decode_rejection_warmup(_worker(None))
|
||||
|
||||
assert calls
|
||||
for kwargs in calls:
|
||||
assert kwargs["draft_sampled"].dtype == torch.int32
|
||||
assert kwargs["pos"].dtype == torch.int64
|
||||
|
||||
|
||||
def test_warmup_index_mapping_dtypes_match_the_runtime_buffers(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
calls = _capture_calls(monkeypatch)
|
||||
spec_decode_rejection_warmup(_worker(None))
|
||||
|
||||
assert calls
|
||||
for kwargs in calls:
|
||||
assert kwargs["idx_mapping"].dtype == torch.int64
|
||||
assert kwargs["expanded_idx_mapping"].dtype == torch.int64
|
||||
assert kwargs["expanded_local_pos"].dtype == torch.int32
|
||||
|
||||
|
||||
def test_warmup_watermark_args_match_the_runtime_kernel_arguments(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
watermark_config = WatermarkConfig(
|
||||
key=MASTER_KEY, algorithm="dual_key_gumbel", context_width=3
|
||||
)
|
||||
calls = _capture_calls(monkeypatch)
|
||||
spec_decode_rejection_warmup(
|
||||
_worker(watermark_config, draft_sample_method="probabilistic")
|
||||
)
|
||||
|
||||
assert calls
|
||||
key_b = derive_watermark_key(MASTER_KEY, b"key_b")
|
||||
for kwargs in calls:
|
||||
contexts = kwargs["contexts"]
|
||||
assert contexts.dtype == torch.int32
|
||||
assert contexts.shape == (3, 3) and contexts.stride() == (3, 1)
|
||||
assert kwargs["watermarking"].dtype == torch.bool
|
||||
assert kwargs["watermark_key"] == key_b
|
||||
assert key_b & 0xFFFFFFFF >= 2**31 and key_b >> 32 >= 2**31
|
||||
|
||||
|
||||
def test_warmup_is_skipped_when_the_recovery_key_cannot_be_derived(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
calls = _capture_calls(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.watermarking.spec_decode.speculative_target_watermark_key",
|
||||
lambda _: (_ for _ in ()).throw(NotImplementedError("no philox")),
|
||||
)
|
||||
spec_decode_rejection_warmup(_worker(WatermarkConfig(key=MASTER_KEY)))
|
||||
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_warmup_draft_logits_match_the_runtime_draft_sample_method(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
calls = _capture_calls(monkeypatch)
|
||||
spec_decode_rejection_warmup(_worker(None))
|
||||
assert calls
|
||||
assert all(kwargs["draft_logits"] is None for kwargs in calls)
|
||||
|
||||
calls = _capture_calls(monkeypatch)
|
||||
spec_decode_rejection_warmup(_worker(None, draft_sample_method="probabilistic"))
|
||||
assert calls
|
||||
assert all(kwargs["draft_logits"] is not None for kwargs in calls)
|
||||
@@ -0,0 +1,300 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The warmup must cover every specialization the sampler will launch.
|
||||
|
||||
``_philox_gumbel_kernel`` is keyed on the Philox key halves, on the logits
|
||||
dtype (model dtype when the sampler passes logits through, fp32 when logits
|
||||
processing copies them) and on whether a skip mask is passed, so a missing
|
||||
combination leaves a request paying the compile this module exists to remove.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.watermarking import WatermarkConfig, derive_watermark_key
|
||||
from vllm.model_executor.warmup.watermark_sample_warmup import watermark_sample_warmup
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.watermarking.factory import create_watermarker
|
||||
from vllm.v1.watermarking.gpu_sampler import GPUWatermarkSampler
|
||||
from vllm.v1.watermarking.spec_decode import create_speculative_target_watermarker
|
||||
|
||||
MASTER_KEY = 20250910
|
||||
KEY_A = derive_watermark_key(MASTER_KEY, b"key_a")
|
||||
KEY_B = derive_watermark_key(MASTER_KEY, b"key_b")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def cuda_alike_platform(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""The warmup returns early off CUDA, so pin the platform it reads.
|
||||
|
||||
Everything here runs on the fake worker's cpu device; without this the
|
||||
tests below would pass only on a CUDA-alike host and silently warm nothing
|
||||
anywhere else.
|
||||
"""
|
||||
monkeypatch.setattr(current_platform, "is_cuda_alike", lambda: True)
|
||||
|
||||
|
||||
def _sampler(
|
||||
watermark_config: WatermarkConfig,
|
||||
speculative: bool,
|
||||
use_fp64_gumbel: bool,
|
||||
) -> GPUWatermarkSampler:
|
||||
"""The sampler ``GPUModelRunner`` builds for this config.
|
||||
|
||||
Mirrors ``vllm/v1/worker/gpu/model_runner.py:473-486``. ``__new__`` skips
|
||||
``Sampler.__init__``, which allocates UVA-backed device buffers; the warmup
|
||||
only reads the three attributes set here.
|
||||
"""
|
||||
watermarker = create_watermarker(watermark_config)
|
||||
if speculative:
|
||||
watermarker = create_speculative_target_watermarker(watermarker)
|
||||
sampler = GPUWatermarkSampler.__new__(GPUWatermarkSampler)
|
||||
sampler.watermarker = watermarker
|
||||
sampler.deduplicate_contexts = watermark_config.deduplicate_contexts
|
||||
sampler.use_fp64_gumbel = use_fp64_gumbel
|
||||
return sampler
|
||||
|
||||
|
||||
def _worker(
|
||||
watermark_config: WatermarkConfig | None,
|
||||
speculative: bool = False,
|
||||
use_fp64_gumbel: bool = False,
|
||||
with_sampler: bool = True,
|
||||
) -> SimpleNamespace:
|
||||
sampler = (
|
||||
_sampler(watermark_config, speculative, use_fp64_gumbel)
|
||||
if watermark_config is not None and with_sampler
|
||||
else None
|
||||
)
|
||||
return SimpleNamespace(
|
||||
device=torch.device("cpu"),
|
||||
model_runner=SimpleNamespace(sampler=sampler),
|
||||
vllm_config=SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
get_vocab_size=lambda: 128,
|
||||
dtype=torch.bfloat16,
|
||||
),
|
||||
watermark_config=watermark_config,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _capture_calls(monkeypatch: pytest.MonkeyPatch) -> list[dict]:
|
||||
calls: list[dict] = []
|
||||
|
||||
def fake_philox_gumbel_sample(logits, contexts, key, **kwargs):
|
||||
calls.append({"logits": logits, "contexts": contexts, "key": key, **kwargs})
|
||||
return torch.zeros(logits.shape[0], dtype=torch.int64)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.worker.gpu.sample.watermark.philox_gumbel_sample",
|
||||
fake_philox_gumbel_sample,
|
||||
)
|
||||
return calls
|
||||
|
||||
|
||||
def _variants(calls: list[dict]) -> set[tuple[int, torch.dtype, bool]]:
|
||||
return {
|
||||
(call["key"], call["logits"].dtype, call.get("skip_mask") is not None)
|
||||
for call in calls
|
||||
}
|
||||
|
||||
|
||||
def test_no_warmup_without_a_watermark_config(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
calls = _capture_calls(monkeypatch)
|
||||
watermark_sample_warmup(_worker(None))
|
||||
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_no_warmup_without_a_watermarked_sampler(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A rank that does not sample launches nothing to warm.
|
||||
|
||||
Only the final pipeline stage of a generative model holds a
|
||||
``GPUWatermarkSampler``; on any other rank the kernel is never launched.
|
||||
"""
|
||||
calls = _capture_calls(monkeypatch)
|
||||
watermark_sample_warmup(
|
||||
_worker(WatermarkConfig(key=MASTER_KEY), with_sampler=False)
|
||||
)
|
||||
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_no_warmup_off_cuda(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""``GumbelWatermarker`` launches the kernel only on a CUDA-alike device.
|
||||
|
||||
Elsewhere -- an XPU worker inherits ``kernel_warmup`` from the CUDA worker
|
||||
-- the sampler takes the torch path, so warming would compile a kernel
|
||||
nothing launches.
|
||||
"""
|
||||
calls = _capture_calls(monkeypatch)
|
||||
monkeypatch.setattr(current_platform, "is_cuda_alike", lambda: False)
|
||||
watermark_sample_warmup(_worker(WatermarkConfig(key=MASTER_KEY)))
|
||||
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_single_key_warms_both_logits_dtypes(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
calls = _capture_calls(monkeypatch)
|
||||
watermark_sample_warmup(_worker(WatermarkConfig(key=MASTER_KEY)))
|
||||
|
||||
assert _variants(calls) == {
|
||||
(MASTER_KEY, torch.bfloat16, True),
|
||||
(MASTER_KEY, torch.float32, True),
|
||||
}
|
||||
assert len(calls) == len(_variants(calls))
|
||||
|
||||
|
||||
def test_dual_key_warms_both_derived_keys(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""``DualKeyGumbelWatermarker.sample`` launches the kernel once per key.
|
||||
|
||||
The halves of a derived key exceed 2**31, so each key is typed i64/i32 by
|
||||
magnitude and a wrong key compiles a specialization the engine never uses.
|
||||
"""
|
||||
calls = _capture_calls(monkeypatch)
|
||||
watermark_sample_warmup(
|
||||
_worker(WatermarkConfig(key=MASTER_KEY, algorithm="dual_key_gumbel"))
|
||||
)
|
||||
|
||||
assert _variants(calls) == {
|
||||
(KEY_A, torch.bfloat16, True),
|
||||
(KEY_A, torch.float32, True),
|
||||
(KEY_B, torch.bfloat16, True),
|
||||
(KEY_B, torch.float32, True),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("alpha,key", [(0.0, KEY_A), (1.0, KEY_B)])
|
||||
def test_dual_key_alpha_bounds_warm_one_key(
|
||||
monkeypatch: pytest.MonkeyPatch, alpha: float, key: int
|
||||
) -> None:
|
||||
calls = _capture_calls(monkeypatch)
|
||||
watermark_sample_warmup(
|
||||
_worker(
|
||||
WatermarkConfig(key=MASTER_KEY, algorithm="dual_key_gumbel", alpha=alpha)
|
||||
)
|
||||
)
|
||||
|
||||
assert _variants(calls) == {
|
||||
(key, torch.bfloat16, True),
|
||||
(key, torch.float32, True),
|
||||
}
|
||||
|
||||
|
||||
def test_speculative_config_warms_the_target_role_only(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Under spec decode the sampler holds ``create_target_watermarker()``.
|
||||
|
||||
Rows without draft tokens sample through it, so the sampler path launches
|
||||
key B alone; the draft key runs in the speculator's own sampler.
|
||||
"""
|
||||
calls = _capture_calls(monkeypatch)
|
||||
watermark_sample_warmup(
|
||||
_worker(
|
||||
WatermarkConfig(key=MASTER_KEY, algorithm="dual_key_gumbel"),
|
||||
speculative=True,
|
||||
)
|
||||
)
|
||||
|
||||
assert _variants(calls) == {
|
||||
(KEY_B, torch.bfloat16, True),
|
||||
(KEY_B, torch.float32, True),
|
||||
}
|
||||
|
||||
|
||||
def test_dedup_none_also_warms_the_mask_free_specialization(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Without deduplication an all-watermarked batch passes no skip mask.
|
||||
|
||||
A later batch mixing in an opted-out request passes one, so both
|
||||
specializations are reachable and both must be warmed.
|
||||
"""
|
||||
calls = _capture_calls(monkeypatch)
|
||||
watermark_sample_warmup(
|
||||
_worker(WatermarkConfig(key=MASTER_KEY, deduplicate_contexts="none"))
|
||||
)
|
||||
|
||||
assert _variants(calls) == {
|
||||
(MASTER_KEY, torch.bfloat16, True),
|
||||
(MASTER_KEY, torch.float32, True),
|
||||
(MASTER_KEY, torch.bfloat16, False),
|
||||
(MASTER_KEY, torch.float32, False),
|
||||
}
|
||||
for call in calls:
|
||||
if call.get("skip_mask") is None:
|
||||
assert "temperatures" not in call
|
||||
|
||||
|
||||
def test_warmup_argument_dtypes_match_the_runtime_buffers(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Contexts are int32, the sampling state int64 indices and fp32 temperatures.
|
||||
|
||||
``GPUWatermarkSampler._get_contexts`` gathers from the int32 request-state
|
||||
token ids, ``expanded_idx_mapping``/``positions``/``seeds`` are int64 and
|
||||
``SamplingStates.temperature`` is fp32; warming other dtypes compiles
|
||||
specializations no engine launches.
|
||||
"""
|
||||
calls = _capture_calls(monkeypatch)
|
||||
watermark_sample_warmup(_worker(WatermarkConfig(key=MASTER_KEY, context_width=3)))
|
||||
|
||||
assert calls
|
||||
for call in calls:
|
||||
num_tokens = call["logits"].shape[0]
|
||||
assert call["contexts"].dtype == torch.int32
|
||||
assert call["contexts"].shape == (num_tokens, 3)
|
||||
assert call["skip_mask"].dtype == torch.bool
|
||||
assert call["expanded_idx_mapping"].dtype == torch.int64
|
||||
assert call["positions"].dtype == torch.int64
|
||||
assert call["seeds"].dtype == torch.int64
|
||||
assert call["temperatures"].dtype == torch.float32
|
||||
|
||||
|
||||
def test_use_fp64_follows_the_path_that_forwards_it(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Only the mixed path forwards the sampler's ``use_fp64_gumbel``.
|
||||
|
||||
``GumbelWatermarker._sample_watermarked`` launches the mask-free variant
|
||||
with the ``use_fp64=False`` default, so warming it with fp64 would compile
|
||||
a specialization the engine never launches and miss the one it does.
|
||||
"""
|
||||
calls = _capture_calls(monkeypatch)
|
||||
watermark_sample_warmup(
|
||||
_worker(
|
||||
WatermarkConfig(key=MASTER_KEY, deduplicate_contexts="none"),
|
||||
use_fp64_gumbel=True,
|
||||
)
|
||||
)
|
||||
|
||||
assert calls
|
||||
for call in calls:
|
||||
assert call["use_fp64"] is (call.get("skip_mask") is not None)
|
||||
|
||||
|
||||
def test_warmup_failure_does_not_abort_startup(
|
||||
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
def failing_philox_gumbel_sample(*args, **kwargs):
|
||||
raise RuntimeError("no triton")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.worker.gpu.sample.watermark.philox_gumbel_sample",
|
||||
failing_philox_gumbel_sample,
|
||||
)
|
||||
with caplog.at_level(logging.WARNING):
|
||||
watermark_sample_warmup(_worker(WatermarkConfig(key=MASTER_KEY)))
|
||||
|
||||
assert "Skipping watermark sampler warmup." in caplog.text
|
||||
+180
-5
@@ -2856,14 +2856,183 @@ def _watermarked_vllm_config() -> VllmConfig:
|
||||
return config
|
||||
|
||||
|
||||
def test_gumbel_watermark_rejects_speculative_decoding():
|
||||
def test_target_only_gumbel_allows_speculative_decoding(caplog_vllm, disable_log_dedup):
|
||||
config = _watermarked_vllm_config()
|
||||
config.speculative_config = SpeculativeConfig(
|
||||
method="ngram",
|
||||
num_speculative_tokens=1,
|
||||
config.watermark_config = WatermarkConfig(
|
||||
key=42, allow_target_only_watermarking=True
|
||||
)
|
||||
config.speculative_config = SimpleNamespace(
|
||||
method="mtp",
|
||||
draft_sample_method="probabilistic",
|
||||
rejection_sample_method="standard",
|
||||
parallel_drafting=False,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="does not support speculative decoding"):
|
||||
with caplog_vllm.at_level(logging.WARNING):
|
||||
config._check_watermarking_unsupported()
|
||||
|
||||
assert "Target-only watermarking leaves accepted draft tokens" in caplog_vllm.text
|
||||
assert "Context deduplication is not supported" in caplog_vllm.text
|
||||
|
||||
|
||||
def test_speculative_watermarking_without_context_dedup_does_not_warn(
|
||||
caplog_vllm, disable_log_dedup
|
||||
):
|
||||
config = _watermarked_vllm_config()
|
||||
config.watermark_config = WatermarkConfig(
|
||||
algorithm="dual_key_gumbel", key=42, deduplicate_contexts="none"
|
||||
)
|
||||
config.speculative_config = SimpleNamespace(
|
||||
method="mtp",
|
||||
draft_sample_method="probabilistic",
|
||||
rejection_sample_method="standard",
|
||||
parallel_drafting=False,
|
||||
)
|
||||
|
||||
with caplog_vllm.at_level(logging.WARNING):
|
||||
config._check_watermarking_unsupported()
|
||||
|
||||
assert "Context deduplication is not supported" not in caplog_vllm.text
|
||||
|
||||
|
||||
def test_gumbel_rejects_speculative_decoding_without_target_only():
|
||||
config = _watermarked_vllm_config()
|
||||
config.watermark_config = WatermarkConfig(
|
||||
algorithm="gumbel", key=42, allow_target_only_watermarking=False
|
||||
)
|
||||
config.speculative_config = SimpleNamespace(
|
||||
method="mtp",
|
||||
draft_sample_method="probabilistic",
|
||||
rejection_sample_method="standard",
|
||||
parallel_drafting=False,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="'gumbel'.*allow_target_only_watermarking"):
|
||||
config._check_watermarking_unsupported()
|
||||
|
||||
|
||||
def test_dual_key_gumbel_warns_that_configured_alpha_is_unused(
|
||||
caplog_vllm, disable_log_dedup
|
||||
):
|
||||
config = _watermarked_vllm_config()
|
||||
config.watermark_config = WatermarkConfig(
|
||||
algorithm="dual_key_gumbel", key=42, alpha=0.25
|
||||
)
|
||||
config.speculative_config = SimpleNamespace(
|
||||
method="mtp",
|
||||
draft_sample_method="probabilistic",
|
||||
rejection_sample_method="standard",
|
||||
parallel_drafting=False,
|
||||
)
|
||||
|
||||
with caplog_vllm.at_level(logging.WARNING):
|
||||
config._check_watermarking_unsupported()
|
||||
|
||||
assert "The configured alpha=0.25 is not used" in caplog_vllm.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("alpha", "speculative"),
|
||||
[(0.1, True), (0.25, False)],
|
||||
ids=["default-alpha", "no-specdec"],
|
||||
)
|
||||
def test_dual_key_gumbel_alpha_warning_is_not_emitted(
|
||||
caplog_vllm, disable_log_dedup, alpha, speculative
|
||||
):
|
||||
config = _watermarked_vllm_config()
|
||||
config.watermark_config = WatermarkConfig(
|
||||
algorithm="dual_key_gumbel", key=42, alpha=alpha
|
||||
)
|
||||
if speculative:
|
||||
config.speculative_config = SimpleNamespace(
|
||||
method="mtp",
|
||||
draft_sample_method="probabilistic",
|
||||
rejection_sample_method="standard",
|
||||
parallel_drafting=False,
|
||||
)
|
||||
|
||||
with caplog_vllm.at_level(logging.WARNING):
|
||||
config._check_watermarking_unsupported()
|
||||
|
||||
assert "is not used" not in caplog_vllm.text
|
||||
|
||||
|
||||
def test_dual_key_gumbel_requires_probabilistic_drafting():
|
||||
config = _watermarked_vllm_config()
|
||||
config.watermark_config = WatermarkConfig(algorithm="dual_key_gumbel", key=42)
|
||||
config.speculative_config = SimpleNamespace(
|
||||
method="mtp",
|
||||
draft_sample_method="greedy",
|
||||
rejection_sample_method="standard",
|
||||
parallel_drafting=False,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="draft_sample_method='probabilistic'"):
|
||||
config._check_watermarking_unsupported()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method", ["eagle", "eagle3", "mtp"])
|
||||
def test_dual_key_gumbel_supports_probabilistic_speculative_decoding(method):
|
||||
config = _watermarked_vllm_config()
|
||||
config.watermark_config = WatermarkConfig(algorithm="dual_key_gumbel", key=42)
|
||||
config.speculative_config = SimpleNamespace(
|
||||
method=method,
|
||||
draft_sample_method="probabilistic",
|
||||
rejection_sample_method="standard",
|
||||
parallel_drafting=False,
|
||||
)
|
||||
|
||||
config._check_watermarking_unsupported()
|
||||
|
||||
|
||||
def test_dual_key_gumbel_supports_dspark():
|
||||
config = _watermarked_vllm_config()
|
||||
config.watermark_config = WatermarkConfig(algorithm="dual_key_gumbel", key=42)
|
||||
config.speculative_config = SimpleNamespace(
|
||||
method="dspark",
|
||||
draft_sample_method="probabilistic",
|
||||
rejection_sample_method="standard",
|
||||
parallel_drafting=True,
|
||||
)
|
||||
|
||||
config._check_watermarking_unsupported()
|
||||
|
||||
|
||||
def test_dual_key_gumbel_rejects_non_autoregressive_speculation():
|
||||
config = _watermarked_vllm_config()
|
||||
config.watermark_config = WatermarkConfig(algorithm="dual_key_gumbel", key=42)
|
||||
config.speculative_config = SimpleNamespace(
|
||||
method="ngram",
|
||||
draft_sample_method="probabilistic",
|
||||
rejection_sample_method="standard",
|
||||
parallel_drafting=False,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="autoregressive model-based"):
|
||||
config._check_watermarking_unsupported()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("overrides", "match"),
|
||||
[
|
||||
({"rejection_sample_method": "synthetic"}, "rejection_sample_method"),
|
||||
({"rejection_sample_method": "block"}, "rejection_sample_method"),
|
||||
({"parallel_drafting": True}, "Parallel speculative drafting"),
|
||||
],
|
||||
)
|
||||
def test_dual_key_gumbel_rejects_incompatible_speculative_modes(overrides, match):
|
||||
config = _watermarked_vllm_config()
|
||||
config.watermark_config = WatermarkConfig(algorithm="dual_key_gumbel", key=42)
|
||||
values = {
|
||||
"method": "mtp",
|
||||
"draft_sample_method": "probabilistic",
|
||||
"rejection_sample_method": "standard",
|
||||
"parallel_drafting": False,
|
||||
**overrides,
|
||||
}
|
||||
config.speculative_config = SimpleNamespace(**values)
|
||||
|
||||
with pytest.raises(ValueError, match=match):
|
||||
config._check_watermarking_unsupported()
|
||||
|
||||
|
||||
@@ -2882,6 +3051,12 @@ def test_watermark_key_must_fit_in_64_bits():
|
||||
WatermarkConfig(key=2**64)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("alpha", [-0.1, 1.1])
|
||||
def test_watermark_alpha_must_be_a_probability(alpha):
|
||||
with pytest.raises(ValidationError):
|
||||
WatermarkConfig(key=42, algorithm="dual_key_gumbel", alpha=alpha)
|
||||
|
||||
|
||||
def test_unknown_watermark_prf_is_rejected():
|
||||
with pytest.raises(ValidationError):
|
||||
pydantic.TypeAdapter(WatermarkConfig).validate_python(
|
||||
|
||||
@@ -6,7 +6,10 @@ import math
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.v1.watermarking import GumbelWatermarker
|
||||
from vllm.v1.watermarking.spec_decode import watermarked_rejection_sample
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
|
||||
from vllm.v1.worker.gpu.sample.watermark import philox_gumbel_sample
|
||||
from vllm.v1.worker.gpu.spec_decode.rejection_sampler_utils import (
|
||||
rejection_sample,
|
||||
)
|
||||
@@ -139,6 +142,93 @@ def _assert_distribution_match(
|
||||
)
|
||||
|
||||
|
||||
def test_watermarked_bonus_uses_target_key():
|
||||
target_logits = torch.randn(16, device="cuda")
|
||||
inputs = _build_rejection_sample_inputs(
|
||||
target_logits,
|
||||
target_logits,
|
||||
num_speculative_steps=1,
|
||||
temperature=1.0,
|
||||
num_trials=1,
|
||||
)
|
||||
draft_token = inputs["draft_sampled"][1]
|
||||
contexts = torch.tensor(
|
||||
[[10, 11], [11, draft_token.item()]], dtype=torch.int64, device="cuda"
|
||||
)
|
||||
|
||||
sampled, num_sampled = watermarked_rejection_sample(
|
||||
**inputs,
|
||||
num_speculative_steps=1,
|
||||
contexts=contexts,
|
||||
watermarking=torch.tensor([True], device="cuda"),
|
||||
watermarker=GumbelWatermarker(key=42, context_width=2),
|
||||
)
|
||||
expected = GumbelWatermarker(key=42, context_width=2).sample(
|
||||
target_logits.unsqueeze(0), contexts[1:], lambda _: None
|
||||
)
|
||||
|
||||
assert num_sampled.item() == 2
|
||||
assert sampled[0, 1] == expected.token_ids[0]
|
||||
|
||||
|
||||
def test_watermarked_recovery_uses_target_key():
|
||||
target_logits = torch.tensor([float("-inf"), 0.0, 1.0, 2.0], device="cuda")
|
||||
draft_logits = torch.tensor([100.0, -100.0, -100.0, -100.0], device="cuda")
|
||||
inputs = _build_rejection_sample_inputs(
|
||||
target_logits,
|
||||
draft_logits,
|
||||
num_speculative_steps=1,
|
||||
temperature=1.0,
|
||||
num_trials=1,
|
||||
)
|
||||
inputs["draft_sampled"][1] = 0
|
||||
contexts = torch.tensor([[10, 11], [11, 0]], dtype=torch.int64, device="cuda")
|
||||
|
||||
sampled, num_sampled = watermarked_rejection_sample(
|
||||
**inputs,
|
||||
num_speculative_steps=1,
|
||||
contexts=contexts,
|
||||
watermarking=torch.tensor([True], device="cuda"),
|
||||
watermarker=GumbelWatermarker(key=42, context_width=2),
|
||||
)
|
||||
expected = GumbelWatermarker(key=42, context_width=2).sample(
|
||||
target_logits.unsqueeze(0), contexts[:1], lambda _: None
|
||||
)
|
||||
|
||||
assert num_sampled.item() == 1
|
||||
assert sampled[0, 0] == expected.token_ids[0]
|
||||
|
||||
|
||||
def test_watermarked_recovery_supports_smaller_draft_vocabulary():
|
||||
target_logits = torch.tensor([float("-inf"), 0.0, 1.0, 2.0, 100.0], device="cuda")
|
||||
draft_logits = torch.tensor([100.0, -100.0, -100.0, -100.0], device="cuda")
|
||||
inputs = _build_rejection_sample_inputs(
|
||||
target_logits[:4],
|
||||
draft_logits,
|
||||
num_speculative_steps=1,
|
||||
temperature=1.0,
|
||||
num_trials=1,
|
||||
)
|
||||
inputs["target_logits"] = target_logits.unsqueeze(0).expand(2, -1).contiguous()
|
||||
inputs["draft_sampled"][1] = 0
|
||||
contexts = torch.tensor([[10, 11], [11, 0]], dtype=torch.int64, device="cuda")
|
||||
watermarker = GumbelWatermarker(key=42, context_width=2)
|
||||
|
||||
sampled, num_sampled = watermarked_rejection_sample(
|
||||
**inputs,
|
||||
num_speculative_steps=1,
|
||||
contexts=contexts,
|
||||
watermarking=torch.tensor([True], device="cuda"),
|
||||
watermarker=watermarker,
|
||||
)
|
||||
expected = watermarker.sample(
|
||||
target_logits[:4].unsqueeze(0), contexts[:1], lambda _: None
|
||||
)
|
||||
|
||||
assert num_sampled.item() == 1
|
||||
assert sampled[0, 0] == expected.token_ids[0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_speculative_steps,temperature",
|
||||
[
|
||||
@@ -747,3 +837,192 @@ def test_chunked_requests_match_full_batch(has_draft_logits: bool):
|
||||
steps = torch.arange(num_speculative_steps + 1, device=device)
|
||||
valid = steps.unsqueeze(0) < num_sampled.unsqueeze(1)
|
||||
assert torch.equal(chunked_sampled[valid], sampled[valid])
|
||||
|
||||
|
||||
def _residual_recovery_logits(
|
||||
inputs: dict,
|
||||
num_sampled: torch.Tensor,
|
||||
vocab_size: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
target_logits = inputs["target_logits"]
|
||||
draft_logits = inputs["draft_logits"]
|
||||
cu_num_logits = inputs["cu_num_logits"]
|
||||
resample_rows = cu_num_logits[:-1].to(torch.int64) + num_sampled.to(torch.int64) - 1
|
||||
req_state_indices = inputs["expanded_idx_mapping"][resample_rows].to(torch.int64)
|
||||
temperatures = inputs["temperature"][req_state_indices]
|
||||
|
||||
is_bonus = resample_rows == cu_num_logits[1:] - 1
|
||||
next_rows = (resample_rows + 1).clamp_max(inputs["draft_sampled"].shape[0] - 1)
|
||||
needs_residual = ~is_bonus & (inputs["draft_sampled"][next_rows] >= 0)
|
||||
|
||||
target_rows = target_logits[resample_rows][:, :vocab_size].float()
|
||||
target_rows = torch.where(target_rows.isnan(), float("-inf"), target_rows)
|
||||
if draft_logits is None:
|
||||
rejected = inputs["draft_sampled"][next_rows].clamp_min(0)
|
||||
one_hot = torch.arange(vocab_size, device=target_rows.device).unsqueeze(
|
||||
0
|
||||
) == rejected.unsqueeze(-1)
|
||||
residual_logits = torch.where(one_hot, float("-inf"), target_rows)
|
||||
else:
|
||||
draft_steps = (num_sampled.to(torch.int64) - 1).clamp_max(
|
||||
draft_logits.shape[1] - 1
|
||||
)
|
||||
draft_rows = draft_logits[req_state_indices, draft_steps][
|
||||
:, :vocab_size
|
||||
].float()
|
||||
draft_rows = torch.where(draft_rows.isnan(), float("-inf"), draft_rows)
|
||||
draft_rows = draft_rows / temperatures.unsqueeze(-1)
|
||||
target_log_probs = torch.log_softmax(target_rows, dim=-1)
|
||||
draft_log_probs = torch.log_softmax(draft_rows, dim=-1)
|
||||
ratio = torch.exp(draft_log_probs - target_log_probs)
|
||||
residual_logits = torch.where(
|
||||
ratio < 1,
|
||||
target_log_probs + torch.log1p(-ratio.clamp_max(1)),
|
||||
float("-inf"),
|
||||
)
|
||||
recovery = torch.where(needs_residual.unsqueeze(-1), residual_logits, target_rows)
|
||||
return recovery, resample_rows
|
||||
|
||||
|
||||
def _watermark_inputs(
|
||||
num_trials: int,
|
||||
num_speculative_steps: int,
|
||||
context_width: int,
|
||||
vocab_size: int,
|
||||
one_hot_draft: bool,
|
||||
temperature: float = 1.0,
|
||||
) -> tuple[dict, torch.Tensor]:
|
||||
device = "cuda"
|
||||
target_logits_1d = torch.randn(vocab_size, device=device) / temperature
|
||||
draft_logits_1d = torch.randn(vocab_size, device=device)
|
||||
inputs = _build_rejection_sample_inputs(
|
||||
target_logits_1d,
|
||||
draft_logits_1d,
|
||||
num_speculative_steps,
|
||||
temperature=temperature,
|
||||
num_trials=num_trials,
|
||||
)
|
||||
if one_hot_draft:
|
||||
inputs["draft_logits"] = None
|
||||
inputs["draft_sampled"].view(num_trials, num_speculative_steps + 1)[::5, 1] = -1
|
||||
num_logits = inputs["target_logits"].shape[0]
|
||||
contexts = torch.randint(
|
||||
0, vocab_size, (num_logits, context_width), dtype=torch.int64, device=device
|
||||
)
|
||||
contexts[::7, 0] = -1
|
||||
return inputs, contexts
|
||||
|
||||
|
||||
@pytest.mark.parametrize("one_hot_draft", [False, True])
|
||||
# Cover one full block and a partial multi-block vocabulary.
|
||||
@pytest.mark.parametrize("vocab_size", [1024, 3500])
|
||||
def test_watermarked_recovery_matches_philox_gumbel_sample(
|
||||
one_hot_draft: bool, vocab_size: int
|
||||
):
|
||||
torch.manual_seed(7)
|
||||
key = 0xDA39A3EE5E6B4B0D
|
||||
context_width = 4
|
||||
inputs, contexts = _watermark_inputs(
|
||||
num_trials=128,
|
||||
num_speculative_steps=2,
|
||||
context_width=context_width,
|
||||
vocab_size=vocab_size,
|
||||
one_hot_draft=one_hot_draft,
|
||||
)
|
||||
watermarker = GumbelWatermarker(key=key, context_width=context_width)
|
||||
|
||||
sampled, num_sampled = watermarked_rejection_sample(
|
||||
**inputs,
|
||||
num_speculative_steps=2,
|
||||
contexts=contexts,
|
||||
watermarking=torch.ones(128, dtype=torch.bool, device="cuda"),
|
||||
watermarker=watermarker,
|
||||
)
|
||||
|
||||
recovery, resample_rows = _residual_recovery_logits(inputs, num_sampled, vocab_size)
|
||||
expected = philox_gumbel_sample(recovery, contexts[resample_rows], key)
|
||||
recovered = sampled[torch.arange(128, device="cuda"), num_sampled.long() - 1]
|
||||
torch.testing.assert_close(recovered, expected, rtol=0, atol=0)
|
||||
|
||||
is_bonus = resample_rows == inputs["cu_num_logits"][1:] - 1
|
||||
next_rows = (resample_rows + 1).clamp_max(inputs["draft_sampled"].shape[0] - 1)
|
||||
is_placeholder = ~is_bonus & (inputs["draft_sampled"][next_rows] < 0)
|
||||
assert int((~is_bonus & ~is_placeholder).sum()) > 0
|
||||
assert int(is_placeholder.sum()) > 0
|
||||
if not one_hot_draft:
|
||||
# One-hot drafts almost never reach the bonus row.
|
||||
assert int(is_bonus.sum()) > 0
|
||||
|
||||
|
||||
def test_watermarked_recovery_leaves_disabled_rows_on_the_stock_draw():
|
||||
torch.manual_seed(11)
|
||||
num_trials = 96
|
||||
vocab_size = 512
|
||||
inputs, contexts = _watermark_inputs(
|
||||
num_trials=num_trials,
|
||||
num_speculative_steps=3,
|
||||
context_width=2,
|
||||
vocab_size=vocab_size,
|
||||
one_hot_draft=False,
|
||||
)
|
||||
watermarking = torch.zeros(num_trials, dtype=torch.bool, device="cuda")
|
||||
watermarking[::2] = True
|
||||
inputs["temperature"][:8] = 0.0
|
||||
|
||||
stock_sampled, stock_num_sampled = rejection_sample(
|
||||
**inputs, num_speculative_steps=3
|
||||
)
|
||||
sampled, num_sampled = watermarked_rejection_sample(
|
||||
**inputs,
|
||||
num_speculative_steps=3,
|
||||
contexts=contexts,
|
||||
watermarking=watermarking,
|
||||
watermarker=GumbelWatermarker(key=99, context_width=2),
|
||||
)
|
||||
|
||||
torch.testing.assert_close(num_sampled, stock_num_sampled, rtol=0, atol=0)
|
||||
rows = torch.arange(num_trials, device="cuda")
|
||||
positions = num_sampled.long() - 1
|
||||
disabled = ~watermarking | (inputs["temperature"] == 0)
|
||||
torch.testing.assert_close(
|
||||
sampled[rows[disabled], positions[disabled]],
|
||||
stock_sampled[rows[disabled], positions[disabled]],
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
for req in range(num_trials):
|
||||
n = int(num_sampled[req]) - 1
|
||||
torch.testing.assert_close(
|
||||
sampled[req, :n], stock_sampled[req, :n], rtol=0, atol=0
|
||||
)
|
||||
assert int(disabled.sum()) > 0 and int((~disabled).sum()) > 0
|
||||
|
||||
|
||||
def test_watermarked_recovery_is_unchanged_by_fp64_gumbel():
|
||||
torch.manual_seed(13)
|
||||
inputs, contexts = _watermark_inputs(
|
||||
num_trials=64,
|
||||
num_speculative_steps=2,
|
||||
context_width=4,
|
||||
vocab_size=2048,
|
||||
one_hot_draft=False,
|
||||
)
|
||||
watermarker = GumbelWatermarker(key=2**63 + 12345, context_width=4)
|
||||
kwargs = dict(
|
||||
num_speculative_steps=2,
|
||||
contexts=contexts,
|
||||
watermarking=torch.ones(64, dtype=torch.bool, device="cuda"),
|
||||
watermarker=watermarker,
|
||||
)
|
||||
sampled32, num_sampled32 = watermarked_rejection_sample(**inputs, **kwargs)
|
||||
sampled64, num_sampled64 = watermarked_rejection_sample(
|
||||
**inputs, **kwargs, use_fp64=True
|
||||
)
|
||||
rows = torch.arange(64, device="cuda")
|
||||
torch.testing.assert_close(num_sampled64, num_sampled32, rtol=0, atol=0)
|
||||
torch.testing.assert_close(
|
||||
sampled64[rows, num_sampled64.long() - 1],
|
||||
sampled32[rows, num_sampled32.long() - 1],
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
|
||||
@@ -5,8 +5,15 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.watermarking import GumbelWatermarkDetector, GumbelWatermarker
|
||||
from vllm.v1.watermarking import (
|
||||
DualKeyGumbelWatermarkDetector,
|
||||
GumbelWatermarkDetector,
|
||||
GumbelWatermarker,
|
||||
derive_watermark_key,
|
||||
)
|
||||
from vllm.v1.watermarking.gumbel import _gamma_survival_integer_shape
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
|
||||
from vllm.v1.worker.gpu.sample.watermark import philox_gumbel_sample
|
||||
|
||||
|
||||
def test_gamma_survival_integer_shape():
|
||||
@@ -34,6 +41,67 @@ def test_detector_deduplicates_context_even_when_target_differs():
|
||||
assert detection.num_scored_tokens == 3
|
||||
|
||||
|
||||
def test_dual_key_detector_scores_each_token_against_both_keys():
|
||||
token_ids = [1, 2, 3, 4, 5]
|
||||
key_a_detector = GumbelWatermarkDetector(
|
||||
key=derive_watermark_key(42, b"key_a"),
|
||||
context_width=1,
|
||||
deduplicate_contexts=False,
|
||||
)
|
||||
key_b_detector = GumbelWatermarkDetector(
|
||||
key=derive_watermark_key(42, b"key_b"),
|
||||
context_width=1,
|
||||
deduplicate_contexts=False,
|
||||
)
|
||||
detector = DualKeyGumbelWatermarkDetector(
|
||||
key=42,
|
||||
context_width=1,
|
||||
deduplicate_contexts=False,
|
||||
alpha=0.5,
|
||||
)
|
||||
|
||||
key_a = key_a_detector.detect(token_ids)
|
||||
key_b = key_b_detector.detect(token_ids)
|
||||
dual = detector.detect(token_ids)
|
||||
|
||||
assert dual.score == pytest.approx((key_a.score + key_b.score) / 2)
|
||||
assert dual.p_value == pytest.approx(
|
||||
_gamma_survival_integer_shape(dual.score * 2, dual.num_scored_tokens * 2)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("alpha", [0.2, 0.7])
|
||||
def test_dual_key_detector_weights_and_recalibrates_scores(alpha: float):
|
||||
token_ids = [1, 2, 3, 4, 5]
|
||||
key_a = GumbelWatermarkDetector(
|
||||
key=derive_watermark_key(42, b"key_a"), deduplicate_contexts=False
|
||||
).detect(token_ids)
|
||||
key_b = GumbelWatermarkDetector(
|
||||
key=derive_watermark_key(42, b"key_b"), deduplicate_contexts=False
|
||||
).detect(token_ids)
|
||||
dual = DualKeyGumbelWatermarkDetector(
|
||||
key=42, deduplicate_contexts=False, alpha=alpha
|
||||
).detect(token_ids)
|
||||
|
||||
variance = (1 - alpha) ** 2 + alpha**2
|
||||
expected_p_value = torch.special.gammaincc(
|
||||
torch.tensor(dual.num_scored_tokens / variance, dtype=torch.float64),
|
||||
torch.tensor(dual.score / variance, dtype=torch.float64),
|
||||
).item()
|
||||
|
||||
assert dual.score == pytest.approx((1 - alpha) * key_a.score + alpha * key_b.score)
|
||||
assert dual.p_value == pytest.approx(expected_p_value)
|
||||
|
||||
|
||||
def test_dual_key_detector_rejects_invalid_alpha():
|
||||
with pytest.raises(ValueError, match="alpha must be between 0 and 1"):
|
||||
DualKeyGumbelWatermarkDetector(key=42, alpha=1.1)
|
||||
|
||||
|
||||
def test_dual_key_detector_default_alpha():
|
||||
assert DualKeyGumbelWatermarkDetector(key=42).alpha == 0.2
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
|
||||
)
|
||||
@@ -45,10 +113,8 @@ def test_fused_watermarker_matches_cpu(key: int, context_width: int):
|
||||
logits = torch.randn(32, 8193)
|
||||
watermarker = GumbelWatermarker(key, context_width)
|
||||
|
||||
expected = watermarker.sample(logits, contexts, lambda values: None).token_ids
|
||||
actual = watermarker.sample(
|
||||
logits.cuda(), contexts.cuda(), lambda values: None
|
||||
).token_ids.cpu()
|
||||
expected = watermarker.sample(logits, contexts).token_ids
|
||||
actual = watermarker.sample(logits.cuda(), contexts.cuda()).token_ids.cpu()
|
||||
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
|
||||
@@ -61,7 +127,7 @@ def test_fused_watermarker_handles_nan_logits():
|
||||
logits = torch.full((2, 1025), float("nan"), device="cuda")
|
||||
watermarker = GumbelWatermarker(key=42, context_width=4)
|
||||
|
||||
token_ids = watermarker.sample(logits, contexts, lambda values: None).token_ids
|
||||
token_ids = watermarker.sample(logits, contexts).token_ids
|
||||
|
||||
assert torch.all((token_ids >= 0) & (token_ids < logits.shape[-1]))
|
||||
|
||||
@@ -76,9 +142,58 @@ def test_fused_watermarker_handles_noncontiguous_inputs():
|
||||
logits = torch.randn(8, 2050, device="cuda")[:, ::2]
|
||||
watermarker = GumbelWatermarker(key=42, context_width=4)
|
||||
|
||||
expected = watermarker.sample(
|
||||
logits.contiguous(), contexts.contiguous(), lambda values: None
|
||||
).token_ids
|
||||
actual = watermarker.sample(logits, contexts, lambda values: None).token_ids
|
||||
expected = watermarker.sample(logits.contiguous(), contexts.contiguous()).token_ids
|
||||
actual = watermarker.sample(logits, contexts).token_ids
|
||||
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
|
||||
)
|
||||
@pytest.mark.parametrize("use_fp64", [False, True])
|
||||
def test_philox_gumbel_sample_skip_mask_matches_separate_samplers(use_fp64: bool):
|
||||
torch.manual_seed(0)
|
||||
logits = torch.randn(8, 8193, device="cuda")
|
||||
context_storage = torch.randint(0, 248320, (8, 8), dtype=torch.int64, device="cuda")
|
||||
contexts = context_storage[:, ::2]
|
||||
repeated_mask = torch.tensor(
|
||||
[False, False, False, True, False, False, True, False], device="cuda"
|
||||
)
|
||||
watermarking = torch.tensor(
|
||||
[True, False, True, True, False, True, True, False], device="cuda"
|
||||
)
|
||||
req_indices = torch.tensor([3, 1, 7, 0, 4, 2, 6, 5], device="cuda")
|
||||
temperatures = torch.ones(8, dtype=torch.float32, device="cuda")
|
||||
temperatures[2] = 0
|
||||
seeds = torch.arange(8, dtype=torch.int64, device="cuda") + 1000
|
||||
positions = torch.arange(8, dtype=torch.int64, device="cuda") + 100
|
||||
watermark_mask = (
|
||||
watermarking[req_indices] & (temperatures[req_indices] != 0) & ~repeated_mask
|
||||
)
|
||||
|
||||
watermarked = philox_gumbel_sample(logits, contexts, 42)
|
||||
ordinary = gumbel_sample(
|
||||
logits,
|
||||
req_indices,
|
||||
temperatures,
|
||||
seeds,
|
||||
positions,
|
||||
apply_temperature=False,
|
||||
is_drafting=False,
|
||||
use_fp64=use_fp64,
|
||||
)
|
||||
expected = torch.where(watermark_mask, watermarked, ordinary)
|
||||
actual = philox_gumbel_sample(
|
||||
logits,
|
||||
contexts,
|
||||
42,
|
||||
skip_mask=~watermark_mask,
|
||||
expanded_idx_mapping=req_indices,
|
||||
temperatures=temperatures,
|
||||
seeds=seeds,
|
||||
positions=positions,
|
||||
use_fp64=use_fp64,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
|
||||
@@ -9,44 +9,98 @@ import torch
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.config.watermarking import WatermarkConfig
|
||||
from vllm.v1.watermarking import create_watermarker
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.watermarking import (
|
||||
DualKeyGumbelWatermarker,
|
||||
SupportsSpeculativeDecoding,
|
||||
create_watermarker,
|
||||
derive_watermark_key,
|
||||
)
|
||||
from vllm.v1.watermarking.gpu_sampler import GPUWatermarkSampler
|
||||
from vllm.v1.watermarking.watermarker import WatermarkSample
|
||||
from vllm.v1.watermarking.gumbel import GumbelWatermarker
|
||||
from vllm.v1.watermarking.spec_decode import (
|
||||
DraftWatermarker,
|
||||
_resolve_watermark_key,
|
||||
create_speculative_draft_watermarker,
|
||||
create_speculative_target_watermarker,
|
||||
speculative_target_watermark_key,
|
||||
)
|
||||
from vllm.v1.watermarking.watermarker import Watermarker, WatermarkSample
|
||||
from vllm.v1.worker.gpu.sample.sampler import Sampler
|
||||
from vllm.v1.worker.gpu.sample.watermark import (
|
||||
philox_gumbel_sample,
|
||||
repeated_context_mask,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.dspark.speculator import DSparkSpeculator
|
||||
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
|
||||
|
||||
|
||||
@pytest.mark.parametrize("algorithm", ["gumbel"])
|
||||
class StubWatermarker(Watermarker):
|
||||
context_width = 1
|
||||
|
||||
def _sample_watermarked(self, logits, contexts):
|
||||
return WatermarkSample(torch.tensor([7, 7]), logits + 10)
|
||||
|
||||
|
||||
def _argmax_sampler(logits: torch.Tensor) -> torch.Tensor:
|
||||
return logits.argmax(dim=-1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("algorithm", ["gumbel", "dual_key_gumbel"])
|
||||
def test_watermarker_contract(algorithm: str):
|
||||
watermarker = create_watermarker(
|
||||
WatermarkConfig(algorithm=algorithm, key=42, context_width=4)
|
||||
)
|
||||
logits = torch.zeros(2, 128)
|
||||
contexts = torch.tensor([[1, 2, 3, 4], [4, 5, 6, 7]])
|
||||
random_sample = lambda sample_logits: sample_logits.argmax(dim=-1)
|
||||
|
||||
first = watermarker.sample(logits, contexts, random_sample)
|
||||
second = watermarker.sample(logits, contexts, random_sample)
|
||||
first = watermarker.sample(logits, contexts, _argmax_sampler)
|
||||
second = watermarker.sample(logits, contexts, _argmax_sampler)
|
||||
|
||||
assert first.token_ids.shape == (2,)
|
||||
assert first.logits.shape == logits.shape
|
||||
assert torch.equal(first.token_ids, second.token_ids)
|
||||
|
||||
|
||||
def test_gumbel_config_warns_about_degenerate_generations(monkeypatch):
|
||||
@pytest.mark.parametrize("algorithm", ["gumbel", "dual_key_gumbel"])
|
||||
@pytest.mark.parametrize(
|
||||
"config_overrides",
|
||||
[
|
||||
{"deduplicate_contexts": "none"},
|
||||
{"deduplicate_contexts_max_history": 1023},
|
||||
],
|
||||
)
|
||||
def test_gumbel_config_warns_when_context_deduplication_is_weak(
|
||||
monkeypatch, algorithm, config_overrides
|
||||
):
|
||||
messages: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
"vllm.config.watermarking.logger.warning_once",
|
||||
lambda message, *, scope: messages.append(message),
|
||||
)
|
||||
|
||||
WatermarkConfig(key=42)
|
||||
WatermarkConfig(key=42, algorithm=algorithm)
|
||||
WatermarkConfig(key=42, algorithm=algorithm, deduplicate_contexts_max_history=1024)
|
||||
WatermarkConfig(key=42, algorithm=algorithm, deduplicate_contexts_max_history=None)
|
||||
WatermarkConfig(key=42, algorithm=algorithm, **config_overrides)
|
||||
|
||||
assert messages == [
|
||||
"Single-key Gumbel-max watermarking may increase the frequency of "
|
||||
"degenerate generations, including repetition loops."
|
||||
(
|
||||
"Gumbel-max watermarking with context deduplication disabled "
|
||||
"or limited to fewer than 1024 positions may increase the frequency of "
|
||||
"degenerate generations, including repetition loops. Use "
|
||||
"deduplicate_contexts='single_turn' or 'all' with "
|
||||
"deduplicate_contexts_max_history at least 1024 or null to mitigate this."
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_context_deduplication_history_can_be_unbounded():
|
||||
config = WatermarkConfig(key=42, deduplicate_contexts_max_history=None)
|
||||
|
||||
assert config.deduplicate_contexts_max_history is None
|
||||
|
||||
|
||||
def test_large_context_width_warns_but_is_allowed():
|
||||
config = WatermarkConfig(key=42, context_width=17)
|
||||
|
||||
@@ -56,6 +110,158 @@ def test_large_context_width_warns_but_is_allowed():
|
||||
assert watermarker.context_width == 17
|
||||
|
||||
|
||||
def test_dual_key_watermarker_uses_domain_separated_keys():
|
||||
config = WatermarkConfig(algorithm="dual_key_gumbel", key=42)
|
||||
|
||||
watermarker = create_watermarker(config)
|
||||
assert isinstance(watermarker, SupportsSpeculativeDecoding)
|
||||
draft = watermarker.draft_watermarker
|
||||
target = watermarker.target_watermarker
|
||||
|
||||
assert isinstance(watermarker, DualKeyGumbelWatermarker)
|
||||
assert watermarker.alpha == 0.1
|
||||
assert draft.prf.key == derive_watermark_key(42, b"key_a")
|
||||
assert target.prf.key == derive_watermark_key(42, b"key_b")
|
||||
assert target.prf.key != draft.prf.key
|
||||
|
||||
|
||||
def test_dual_key_watermarker_routes_tokens_with_alpha():
|
||||
watermarker = DualKeyGumbelWatermarker(key=42, context_width=2, alpha=0.25)
|
||||
logits = torch.zeros(2, 16)
|
||||
contexts = torch.tensor([[1, 2], [3, 4]])
|
||||
key_a = watermarker.draft_watermarker.sample(logits, contexts, lambda _: None)
|
||||
key_b = watermarker.target_watermarker.sample(logits, contexts, lambda _: None)
|
||||
|
||||
def route(routing_logits):
|
||||
torch.testing.assert_close(
|
||||
routing_logits.softmax(dim=-1),
|
||||
torch.tensor([[0.75, 0.25], [0.75, 0.25]]),
|
||||
)
|
||||
return torch.tensor([0, 1])
|
||||
|
||||
sampled = watermarker.sample(logits, contexts, route)
|
||||
|
||||
assert torch.equal(
|
||||
sampled.token_ids, torch.stack([key_a.token_ids[0], key_b.token_ids[1]])
|
||||
)
|
||||
|
||||
|
||||
def test_dual_key_routing_logits_are_cached():
|
||||
alpha = 0.25
|
||||
watermarker = DualKeyGumbelWatermarker(key=42, context_width=2, alpha=alpha)
|
||||
logits = torch.zeros(2, 16)
|
||||
contexts = torch.tensor([[1, 2], [3, 4]])
|
||||
recorded: list[torch.Tensor] = []
|
||||
|
||||
def route(routing_logits):
|
||||
recorded.append(routing_logits)
|
||||
return torch.tensor([0, 1])
|
||||
|
||||
watermarker.sample(logits, contexts, route)
|
||||
watermarker.sample(logits, contexts, route)
|
||||
|
||||
expected = torch.tensor([1 - alpha, alpha], dtype=torch.float32).log()
|
||||
expected = expected.expand(2, -1)
|
||||
assert len(recorded) == 2
|
||||
assert all(torch.equal(seen, expected) for seen in recorded)
|
||||
assert len(watermarker._routing_logits_cache) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("alpha", [0.0, 0.25, 1.0])
|
||||
def test_dual_key_watermarker_skips_masked_rows(alpha: float):
|
||||
watermarker = DualKeyGumbelWatermarker(key=42, context_width=2, alpha=alpha)
|
||||
torch.manual_seed(0)
|
||||
logits = torch.randn(4, 32)
|
||||
contexts = torch.tensor([[1, 2], [3, 4], [5, 6], [7, 8]])
|
||||
skip_mask = torch.tensor([True, False, True, False])
|
||||
|
||||
unmasked = watermarker.sample(logits, contexts, _argmax_sampler)
|
||||
masked = watermarker.sample(logits, contexts, _argmax_sampler, skip_mask)
|
||||
|
||||
ordinary = _argmax_sampler(logits)
|
||||
assert torch.equal(masked.token_ids[skip_mask], ordinary[skip_mask])
|
||||
assert torch.equal(masked.token_ids[~skip_mask], unmasked.token_ids[~skip_mask])
|
||||
assert not torch.equal(unmasked.token_ids[skip_mask], ordinary[skip_mask])
|
||||
|
||||
|
||||
def test_dual_key_watermarker_routing_requires_a_random_sampler():
|
||||
watermarker = DualKeyGumbelWatermarker(key=42, context_width=2, alpha=0.25)
|
||||
|
||||
with pytest.raises(ValueError, match="random sampler"):
|
||||
watermarker.sample(torch.zeros(2, 16), torch.tensor([[1, 2], [3, 4]]))
|
||||
|
||||
|
||||
def test_speculative_decoding_uses_fixed_dual_key_roles():
|
||||
watermarker = create_watermarker(
|
||||
WatermarkConfig(algorithm="dual_key_gumbel", key=42, alpha=0.25)
|
||||
)
|
||||
|
||||
target = create_speculative_target_watermarker(watermarker)
|
||||
draft = create_speculative_draft_watermarker(
|
||||
watermarker,
|
||||
max_num_reqs=1,
|
||||
device=torch.device("cpu"),
|
||||
allow_target_only=False,
|
||||
)
|
||||
|
||||
assert target.prf.key == derive_watermark_key(42, b"key_b")
|
||||
assert draft is not None
|
||||
assert draft.watermarker.prf.key == derive_watermark_key(42, b"key_a")
|
||||
|
||||
|
||||
def test_recovery_key_rejects_the_unsplit_dual_key_watermarker():
|
||||
watermarker = create_watermarker(
|
||||
WatermarkConfig(algorithm="dual_key_gumbel", key=42, alpha=0.25)
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="keys the target role separately"):
|
||||
_resolve_watermark_key(watermarker)
|
||||
|
||||
target = create_speculative_target_watermarker(watermarker)
|
||||
assert _resolve_watermark_key(target) == derive_watermark_key(42, b"key_b")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("algorithm", ["gumbel", "dual_key_gumbel"])
|
||||
def test_config_key_resolution_matches_the_model_runner(algorithm: str):
|
||||
config = WatermarkConfig(algorithm=algorithm, key=42, alpha=0.25)
|
||||
runtime_key = _resolve_watermark_key(
|
||||
create_speculative_target_watermarker(create_watermarker(config))
|
||||
)
|
||||
|
||||
assert speculative_target_watermark_key(config) == runtime_key
|
||||
|
||||
|
||||
def test_config_key_resolution_returns_none_when_watermarking_is_disabled():
|
||||
assert speculative_target_watermark_key(None) is None
|
||||
|
||||
|
||||
def test_target_only_speculative_watermarking_skips_draft_watermarker():
|
||||
watermarker = create_watermarker(WatermarkConfig(algorithm="gumbel", key=42))
|
||||
|
||||
assert not isinstance(watermarker, SupportsSpeculativeDecoding)
|
||||
with pytest.raises(ValueError, match="does not support speculative decoding"):
|
||||
create_speculative_draft_watermarker(
|
||||
watermarker,
|
||||
max_num_reqs=1,
|
||||
device=torch.device("cpu"),
|
||||
allow_target_only=False,
|
||||
)
|
||||
assert (
|
||||
create_speculative_draft_watermarker(
|
||||
watermarker,
|
||||
max_num_reqs=1,
|
||||
device=torch.device("cpu"),
|
||||
allow_target_only=True,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_dual_key_derivation_is_stable():
|
||||
assert derive_watermark_key(32, b"key_a") == 16368605726115524094
|
||||
assert derive_watermark_key(32, b"key_b") == 4799302812959726346
|
||||
|
||||
|
||||
def test_sampling_params_can_disable_watermarking():
|
||||
assert SamplingParams().watermarking
|
||||
assert not SamplingParams.from_optional(watermarking=False).watermarking
|
||||
@@ -83,14 +289,9 @@ def test_gpu_sampler_warns_when_watermarking_is_enabled_for_greedy(monkeypatch):
|
||||
|
||||
|
||||
def test_gpu_sampler_respects_mixed_request_watermarking(monkeypatch):
|
||||
class StubWatermarker:
|
||||
context_width = 1
|
||||
|
||||
def sample(self, logits, contexts, random_sample):
|
||||
return WatermarkSample(torch.tensor([7, 7]), logits + 10)
|
||||
|
||||
sampler = object.__new__(GPUWatermarkSampler)
|
||||
sampler.watermarker = StubWatermarker()
|
||||
sampler.deduplicate_contexts = "single_turn"
|
||||
sampler.watermarking = SimpleNamespace(
|
||||
np=np.array([True, False]), gpu=torch.tensor([True, False])
|
||||
)
|
||||
@@ -102,8 +303,11 @@ def test_gpu_sampler_respects_mixed_request_watermarking(monkeypatch):
|
||||
sampler._get_contexts = lambda expanded_idx_mapping: torch.zeros(
|
||||
2, 1, dtype=torch.int64
|
||||
)
|
||||
sampler._get_repeated_contexts = lambda expanded_idx_mapping, contexts: torch.zeros(
|
||||
2, dtype=torch.bool
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.watermarking.gpu_sampler.gumbel_sample",
|
||||
"vllm.v1.watermarking.watermarker.gumbel_sample",
|
||||
lambda *args, **kwargs: torch.tensor([3, 4]),
|
||||
)
|
||||
logits = torch.zeros(2, 8)
|
||||
@@ -123,11 +327,524 @@ def test_gpu_sampler_respects_mixed_request_watermarking(monkeypatch):
|
||||
assert torch.equal(output_logits[1], logits[1])
|
||||
|
||||
|
||||
def test_gpu_sampler_skips_watermarking_for_repeated_contexts(monkeypatch):
|
||||
sampler = object.__new__(GPUWatermarkSampler)
|
||||
sampler.watermarker = StubWatermarker()
|
||||
sampler.deduplicate_contexts = "single_turn"
|
||||
sampler.watermarking = SimpleNamespace(
|
||||
np=np.array([True, True]), gpu=torch.tensor([True, True])
|
||||
)
|
||||
sampler.sampling_states = SimpleNamespace(
|
||||
temperature=SimpleNamespace(np=np.ones(2), gpu=torch.ones(2)),
|
||||
seeds=SimpleNamespace(gpu=torch.zeros(2, dtype=torch.int64)),
|
||||
)
|
||||
sampler.use_fp64_gumbel = False
|
||||
sampler._get_contexts = lambda expanded_idx_mapping: torch.zeros(
|
||||
2, 1, dtype=torch.int64
|
||||
)
|
||||
sampler._get_repeated_contexts = lambda expanded_idx_mapping, contexts: (
|
||||
torch.tensor([True, False])
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.watermarking.watermarker.gumbel_sample",
|
||||
lambda *args, **kwargs: torch.tensor([3, 4]),
|
||||
)
|
||||
logits = torch.zeros(2, 8)
|
||||
|
||||
sampled, output_logits = sampler._sample_random(
|
||||
logits,
|
||||
torch.tensor([0, 1]),
|
||||
np.array([0, 1]),
|
||||
torch.zeros(2, dtype=torch.int64),
|
||||
None,
|
||||
None,
|
||||
False,
|
||||
)
|
||||
|
||||
assert torch.equal(sampled, torch.tensor([3, 7]))
|
||||
assert torch.equal(output_logits[0], logits[0])
|
||||
assert torch.equal(output_logits[1], torch.full((8,), 10.0))
|
||||
|
||||
|
||||
def test_gpu_sampler_can_disable_context_deduplication(monkeypatch):
|
||||
sampler = object.__new__(GPUWatermarkSampler)
|
||||
sampler.watermarker = StubWatermarker()
|
||||
sampler.deduplicate_contexts = "none"
|
||||
sampler.watermarking = SimpleNamespace(
|
||||
np=np.array([True, True]), gpu=torch.tensor([True, True])
|
||||
)
|
||||
sampler.sampling_states = SimpleNamespace(
|
||||
temperature=SimpleNamespace(np=np.ones(2), gpu=torch.ones(2)),
|
||||
seeds=SimpleNamespace(gpu=torch.zeros(2, dtype=torch.int64)),
|
||||
)
|
||||
sampler.use_fp64_gumbel = False
|
||||
sampler._get_contexts = lambda expanded_idx_mapping: torch.zeros(
|
||||
2, 1, dtype=torch.int64
|
||||
)
|
||||
sampler._get_repeated_contexts = lambda *args: pytest.fail(
|
||||
"context deduplication should not run"
|
||||
)
|
||||
logits = torch.zeros(2, 8)
|
||||
|
||||
sampled, output_logits = sampler._sample_random(
|
||||
logits,
|
||||
torch.tensor([0, 1]),
|
||||
np.array([0, 1]),
|
||||
torch.zeros(2, dtype=torch.int64),
|
||||
None,
|
||||
None,
|
||||
False,
|
||||
)
|
||||
|
||||
assert torch.equal(sampled, torch.tensor([7, 7]))
|
||||
assert torch.equal(output_logits, torch.full((2, 8), 10.0))
|
||||
|
||||
|
||||
def test_repeated_context_mask_ignores_prompt_tokens():
|
||||
all_token_ids = torch.tensor(
|
||||
[
|
||||
[8, 9, 1, 2, 1, 2],
|
||||
[3, 4, 1, 2, 3, 4],
|
||||
[0, 0, 0, 0, 0, 0],
|
||||
],
|
||||
dtype=torch.int32,
|
||||
)
|
||||
req_indices = torch.tensor([0, 1, -1])
|
||||
prompt_lens = torch.tensor([2, 2, 0])
|
||||
total_lens = torch.tensor([6, 6, 0])
|
||||
contexts = torch.tensor([[1, 2], [3, 4], [-1, -1]])
|
||||
|
||||
repeated = repeated_context_mask(
|
||||
all_token_ids,
|
||||
req_indices,
|
||||
prompt_lens,
|
||||
total_lens,
|
||||
contexts,
|
||||
)
|
||||
|
||||
assert torch.equal(repeated, torch.tensor([True, False, False]))
|
||||
|
||||
|
||||
def test_repeated_context_mask_can_include_prompt_tokens():
|
||||
all_token_ids = torch.tensor([[1, 2, 3, 4, 5, 6, 1, 2, 3, 4]])
|
||||
req_indices = torch.tensor([0])
|
||||
prompt_lens = torch.tensor([4])
|
||||
total_lens = torch.tensor([10])
|
||||
contexts = torch.tensor([[1, 2, 3, 4]])
|
||||
|
||||
single_turn = repeated_context_mask(
|
||||
all_token_ids, req_indices, prompt_lens, total_lens, contexts
|
||||
)
|
||||
all_history = repeated_context_mask(
|
||||
all_token_ids,
|
||||
req_indices,
|
||||
prompt_lens,
|
||||
total_lens,
|
||||
contexts,
|
||||
include_prompt=True,
|
||||
)
|
||||
|
||||
assert not single_turn.item()
|
||||
assert all_history.item()
|
||||
|
||||
|
||||
def test_repeated_context_mask_can_skip_partial_contexts():
|
||||
all_token_ids = torch.tensor([[10, 11, 12, 13, 1, 2, 3, 4]], dtype=torch.int32)
|
||||
req_indices = torch.tensor([0])
|
||||
prompt_lens = torch.tensor([4])
|
||||
contexts = torch.tensor([[-1, -1, -1, 1]])
|
||||
|
||||
partial = repeated_context_mask(
|
||||
all_token_ids,
|
||||
req_indices,
|
||||
prompt_lens,
|
||||
torch.tensor([5]),
|
||||
contexts,
|
||||
include_prompt=True,
|
||||
skip_partial_context=True,
|
||||
)
|
||||
complete = repeated_context_mask(
|
||||
all_token_ids,
|
||||
req_indices,
|
||||
prompt_lens,
|
||||
torch.tensor([8]),
|
||||
torch.tensor([[1, 2, 3, 4]]),
|
||||
include_prompt=True,
|
||||
skip_partial_context=True,
|
||||
)
|
||||
|
||||
assert partial.item()
|
||||
assert not complete.item()
|
||||
|
||||
|
||||
def test_gpu_sampler_contexts_are_completion_local_for_all_scopes():
|
||||
sampler = object.__new__(GPUWatermarkSampler)
|
||||
sampler.watermarker = SimpleNamespace(context_width=4)
|
||||
sampler.req_states = SimpleNamespace(
|
||||
all_token_ids=SimpleNamespace(gpu=torch.tensor([[10, 11, 12, 13, 1, 2]])),
|
||||
prompt_len=SimpleNamespace(gpu=torch.tensor([4])),
|
||||
total_len=SimpleNamespace(gpu=torch.tensor([6])),
|
||||
)
|
||||
request_indices = torch.tensor([0])
|
||||
|
||||
sampler.deduplicate_contexts = "single_turn"
|
||||
single_turn = sampler._get_contexts(request_indices)
|
||||
sampler.deduplicate_contexts = "all"
|
||||
all_history = sampler._get_contexts(request_indices)
|
||||
|
||||
assert torch.equal(single_turn, torch.tensor([[-1, -1, 1, 2]]))
|
||||
assert torch.equal(all_history, single_turn)
|
||||
|
||||
|
||||
def test_gpu_sampler_all_history_skips_partial_context():
|
||||
sampler = object.__new__(GPUWatermarkSampler)
|
||||
sampler.watermarker = SimpleNamespace(context_width=4)
|
||||
sampler.req_states = SimpleNamespace(
|
||||
all_token_ids=SimpleNamespace(gpu=torch.tensor([[10, 11, 12, 13, 1, 2]])),
|
||||
prompt_len=SimpleNamespace(gpu=torch.tensor([4])),
|
||||
total_len=SimpleNamespace(gpu=torch.tensor([6])),
|
||||
)
|
||||
sampler.deduplicate_contexts_max_history = None
|
||||
request_indices = torch.tensor([0])
|
||||
contexts = sampler._get_contexts(request_indices)
|
||||
|
||||
sampler.deduplicate_contexts = "single_turn"
|
||||
single_turn = sampler._get_repeated_contexts(request_indices, contexts)
|
||||
sampler.deduplicate_contexts = "all"
|
||||
all_history = sampler._get_repeated_contexts(request_indices, contexts)
|
||||
|
||||
assert not single_turn.item()
|
||||
assert all_history.item()
|
||||
|
||||
|
||||
def test_repeated_context_mask_respects_max_history():
|
||||
all_token_ids = torch.tensor([[1, 2, 3, 4, 5, 6, 1, 2, 3, 4]])
|
||||
req_indices = torch.tensor([0])
|
||||
prompt_lens = torch.tensor([0])
|
||||
total_lens = torch.tensor([10])
|
||||
contexts = torch.tensor([[1, 2, 3, 4]])
|
||||
|
||||
full_history = repeated_context_mask(
|
||||
all_token_ids, req_indices, prompt_lens, total_lens, contexts
|
||||
)
|
||||
last_six = repeated_context_mask(
|
||||
all_token_ids, req_indices, prompt_lens, total_lens, contexts, max_history=6
|
||||
)
|
||||
last_five = repeated_context_mask(
|
||||
all_token_ids, req_indices, prompt_lens, total_lens, contexts, max_history=5
|
||||
)
|
||||
|
||||
assert full_history.item()
|
||||
assert last_six.item()
|
||||
assert not last_five.item()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("max_history", [0, -1])
|
||||
def test_repeated_context_mask_rejects_non_positive_max_history(max_history: int):
|
||||
with pytest.raises(ValueError, match="max_history must be positive or None"):
|
||||
repeated_context_mask(
|
||||
torch.tensor([[1, 2, 3, 4]], dtype=torch.int32),
|
||||
torch.tensor([0], dtype=torch.int32),
|
||||
torch.tensor([0], dtype=torch.int32),
|
||||
torch.tensor([4], dtype=torch.int32),
|
||||
torch.tensor([[1, 2, 3, 4]], dtype=torch.int32),
|
||||
max_history=max_history,
|
||||
)
|
||||
|
||||
|
||||
def test_gpu_sampler_wires_request_history_scope():
|
||||
sampler = object.__new__(GPUWatermarkSampler)
|
||||
sampler.req_states = SimpleNamespace(
|
||||
all_token_ids=SimpleNamespace(
|
||||
gpu=torch.tensor([[1, 2, 3, 4, 5, 6, 1, 2, 3, 4]])
|
||||
),
|
||||
prompt_len=SimpleNamespace(gpu=torch.tensor([4])),
|
||||
total_len=SimpleNamespace(gpu=torch.tensor([10])),
|
||||
)
|
||||
sampler.deduplicate_contexts_max_history = None
|
||||
request_indices = torch.tensor([0])
|
||||
contexts = torch.tensor([[1, 2, 3, 4]])
|
||||
|
||||
sampler.deduplicate_contexts = "single_turn"
|
||||
single_turn = sampler._get_repeated_contexts(request_indices, contexts)
|
||||
sampler.deduplicate_contexts = "all"
|
||||
all_history = sampler._get_repeated_contexts(request_indices, contexts)
|
||||
|
||||
assert not single_turn.item()
|
||||
assert all_history.item()
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
|
||||
)
|
||||
@pytest.mark.parametrize("max_history", [5, 6])
|
||||
@pytest.mark.parametrize("include_prompt", [False, True])
|
||||
def test_repeated_context_mask_max_history_accelerator_parity(
|
||||
max_history: int, include_prompt: bool
|
||||
):
|
||||
inputs = (
|
||||
torch.tensor([[1, 2, 3, 4, 5, 6, 1, 2, 3, 4]]),
|
||||
torch.tensor([0]),
|
||||
torch.tensor([0]),
|
||||
torch.tensor([10]),
|
||||
torch.tensor([[1, 2, 3, 4]]),
|
||||
)
|
||||
|
||||
expected = repeated_context_mask(
|
||||
*inputs, max_history=max_history, include_prompt=include_prompt
|
||||
)
|
||||
actual = repeated_context_mask(
|
||||
*(value.cuda() for value in inputs),
|
||||
max_history=max_history,
|
||||
include_prompt=include_prompt,
|
||||
).cpu()
|
||||
|
||||
assert torch.equal(actual, expected)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
|
||||
)
|
||||
def test_repeated_context_mask_partial_context_accelerator_parity():
|
||||
inputs = (
|
||||
torch.tensor([[10, 11, 12, 13, 1]], dtype=torch.int32),
|
||||
torch.tensor([0], dtype=torch.int32),
|
||||
torch.tensor([4], dtype=torch.int32),
|
||||
torch.tensor([5], dtype=torch.int32),
|
||||
torch.tensor([[-1, -1, -1, 1]], dtype=torch.int64),
|
||||
)
|
||||
|
||||
expected = repeated_context_mask(
|
||||
*inputs, include_prompt=True, skip_partial_context=True
|
||||
)
|
||||
actual = repeated_context_mask(
|
||||
*(value.cuda() for value in inputs),
|
||||
include_prompt=True,
|
||||
skip_partial_context=True,
|
||||
).cpu()
|
||||
|
||||
assert torch.equal(actual, expected)
|
||||
|
||||
|
||||
def _unaligned_max_history_inputs() -> tuple[torch.Tensor, ...]:
|
||||
"""One row whose only repeated context ends at position 680.
|
||||
|
||||
680 is not a multiple of the kernel's 512-position block, so the scan window
|
||||
selected by ``max_history`` starts in the middle of a block.
|
||||
"""
|
||||
history = list(range(1_000, 2_200))
|
||||
history[676:680] = [1, 2, 3, 4]
|
||||
return (
|
||||
torch.tensor([history], dtype=torch.int32),
|
||||
torch.tensor([0], dtype=torch.int32),
|
||||
torch.tensor([0], dtype=torch.int32),
|
||||
torch.tensor([len(history)], dtype=torch.int32),
|
||||
torch.tensor([[1, 2, 3, 4]], dtype=torch.int32),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"max_history,expected", [(1200 - 680, True), (1200 - 680 - 1, False)]
|
||||
)
|
||||
def test_repeated_context_mask_unaligned_max_history_accelerator_parity(
|
||||
max_history: int, expected: bool
|
||||
):
|
||||
inputs = _unaligned_max_history_inputs()
|
||||
|
||||
reference = repeated_context_mask(*inputs, max_history=max_history)
|
||||
actual = repeated_context_mask(
|
||||
*(value.cuda() for value in inputs), max_history=max_history
|
||||
).cpu()
|
||||
|
||||
assert torch.equal(reference, torch.tensor([expected]))
|
||||
assert torch.equal(actual, reference)
|
||||
|
||||
|
||||
def _repeated_context_inputs(
|
||||
context_width: int, device: str = "cpu"
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
"""Build repeated, unique, and padded request rows for mask tests."""
|
||||
prompt = [101, 102, 103, 104]
|
||||
repeated_output = [*range(1, context_width + 1)] * 2
|
||||
unique_output = [*range(1, context_width + 2)]
|
||||
max_len = len(prompt) + len(repeated_output)
|
||||
rows = [
|
||||
prompt + repeated_output,
|
||||
prompt + unique_output,
|
||||
[],
|
||||
]
|
||||
all_token_ids = torch.zeros((3, max_len), dtype=torch.int32, device=device)
|
||||
for row, token_ids in enumerate(rows):
|
||||
all_token_ids[row, : len(token_ids)] = torch.tensor(
|
||||
token_ids, dtype=torch.int32, device=device
|
||||
)
|
||||
return (
|
||||
all_token_ids,
|
||||
torch.tensor([0, 1, -1], dtype=torch.int32, device=device),
|
||||
torch.tensor([len(prompt), len(prompt), 0], dtype=torch.int32, device=device),
|
||||
torch.tensor([len(rows[0]), len(rows[1]), 0], dtype=torch.int32, device=device),
|
||||
torch.tensor(
|
||||
[
|
||||
repeated_output[-context_width:],
|
||||
unique_output[-context_width:],
|
||||
[-1] * context_width,
|
||||
],
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("context_width", [1, 3, 4, 16, 17])
|
||||
def test_repeated_context_mask(context_width: int):
|
||||
repeated = repeated_context_mask(*_repeated_context_inputs(context_width))
|
||||
|
||||
assert torch.equal(repeated, torch.tensor([True, False, False]))
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
|
||||
)
|
||||
@pytest.mark.parametrize("context_width", [1, 3, 4, 16, 17])
|
||||
def test_repeated_context_mask_accelerator_parity(context_width: int):
|
||||
cpu_inputs = _repeated_context_inputs(context_width)
|
||||
accelerator_inputs = list(_repeated_context_inputs(context_width, "cuda"))
|
||||
contexts = accelerator_inputs[-1]
|
||||
storage = torch.empty(
|
||||
contexts.shape[0], contexts.shape[1] * 2, dtype=contexts.dtype, device="cuda"
|
||||
)
|
||||
storage[:, ::2] = contexts
|
||||
accelerator_inputs[-1] = storage[:, ::2]
|
||||
|
||||
expected = repeated_context_mask(*cpu_inputs)
|
||||
actual = repeated_context_mask(*accelerator_inputs).cpu()
|
||||
|
||||
assert torch.equal(actual, expected)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
|
||||
)
|
||||
def test_repeated_context_mask_scans_multiple_blocks():
|
||||
context = [1, 2, 3, 4]
|
||||
repeated_output = [*range(10_000, 11_030), *context, 99, *context]
|
||||
unique_output = list(range(20_000, 21_039))
|
||||
prompt = [101, 102, 103, 104]
|
||||
rows = [prompt + repeated_output, prompt + unique_output]
|
||||
all_token_ids = torch.zeros((2, len(rows[0])), dtype=torch.int32)
|
||||
for row, token_ids in enumerate(rows):
|
||||
all_token_ids[row, : len(token_ids)] = torch.tensor(token_ids)
|
||||
inputs = (
|
||||
all_token_ids,
|
||||
torch.tensor([0, 1], dtype=torch.int32),
|
||||
torch.tensor([len(prompt), len(prompt)], dtype=torch.int32),
|
||||
torch.tensor([len(rows[0]), len(rows[1])], dtype=torch.int32),
|
||||
torch.tensor([context, unique_output[-4:]], dtype=torch.int64),
|
||||
)
|
||||
|
||||
expected = repeated_context_mask(*inputs)
|
||||
actual = repeated_context_mask(*(value.cuda() for value in inputs)).cpu()
|
||||
|
||||
assert torch.equal(expected, torch.tensor([True, False]))
|
||||
assert torch.equal(actual, expected)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
|
||||
)
|
||||
def test_repeated_context_mask_last_request_at_capacity():
|
||||
max_model_len = 1024
|
||||
all_token_ids = torch.arange(
|
||||
2 * max_model_len, dtype=torch.int32, device="cuda"
|
||||
).reshape(2, max_model_len)
|
||||
inputs = (
|
||||
all_token_ids,
|
||||
torch.tensor([1], dtype=torch.int32, device="cuda"),
|
||||
torch.tensor([0, 0], dtype=torch.int32, device="cuda"),
|
||||
torch.tensor([max_model_len, max_model_len], dtype=torch.int32, device="cuda"),
|
||||
all_token_ids[1, -4:].to(torch.int64).unsqueeze(0),
|
||||
)
|
||||
|
||||
repeated = repeated_context_mask(*inputs)
|
||||
|
||||
assert not repeated.item()
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
|
||||
)
|
||||
def test_gpu_sampler_uses_fused_gumbel_for_repeated_contexts():
|
||||
device = torch.device("cuda")
|
||||
logits = torch.tensor([[0.0, 1.0, 2.0, 3.0], [3.0, 2.0, 1.0, 0.0]], device=device)
|
||||
all_token_ids = torch.tensor(
|
||||
[[101, 102, 1, 2, 1, 2], [101, 102, 3, 4, 5, 6]],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
request_indices = torch.tensor([0, 1], dtype=torch.int64, device=device)
|
||||
request_indices_np = np.array([0, 1])
|
||||
temperatures = torch.ones(2, device=device)
|
||||
seeds = torch.tensor([11, 22], dtype=torch.int64, device=device)
|
||||
positions = torch.tensor([6, 6], dtype=torch.int64, device=device)
|
||||
|
||||
sampler = object.__new__(GPUWatermarkSampler)
|
||||
sampler.watermarker = GumbelWatermarker(key=42, context_width=1)
|
||||
sampler.deduplicate_contexts = "single_turn"
|
||||
sampler.deduplicate_contexts_max_history = None
|
||||
sampler.watermarking = SimpleNamespace(
|
||||
np=np.ones(2, dtype=bool), gpu=torch.ones(2, dtype=torch.bool, device=device)
|
||||
)
|
||||
sampler.sampling_states = SimpleNamespace(
|
||||
temperature=SimpleNamespace(np=np.ones(2), gpu=temperatures),
|
||||
seeds=SimpleNamespace(gpu=seeds),
|
||||
)
|
||||
sampler.req_states = SimpleNamespace(
|
||||
all_token_ids=SimpleNamespace(gpu=all_token_ids),
|
||||
prompt_len=SimpleNamespace(
|
||||
gpu=torch.tensor([2, 2], dtype=torch.int32, device=device)
|
||||
),
|
||||
total_len=SimpleNamespace(
|
||||
gpu=torch.tensor([6, 6], dtype=torch.int32, device=device)
|
||||
),
|
||||
)
|
||||
sampler.use_fp64_gumbel = False
|
||||
|
||||
contexts = sampler._get_contexts(request_indices)
|
||||
repeated = sampler._get_repeated_contexts(request_indices, contexts)
|
||||
expected = philox_gumbel_sample(
|
||||
logits,
|
||||
contexts,
|
||||
42,
|
||||
skip_mask=repeated,
|
||||
expanded_idx_mapping=request_indices,
|
||||
temperatures=temperatures,
|
||||
seeds=seeds,
|
||||
positions=positions,
|
||||
)
|
||||
|
||||
actual, output_logits = sampler._sample_random(
|
||||
logits,
|
||||
request_indices,
|
||||
request_indices_np,
|
||||
positions,
|
||||
None,
|
||||
None,
|
||||
False,
|
||||
)
|
||||
|
||||
assert torch.equal(repeated, torch.tensor([True, False], device=device))
|
||||
assert torch.equal(actual, expected)
|
||||
assert output_logits is logits
|
||||
|
||||
|
||||
def test_gpu_sampler_skips_watermarking_for_greedy_batch(monkeypatch):
|
||||
class StubWatermarker:
|
||||
context_width = 1
|
||||
|
||||
def sample(self, logits, contexts, random_sample):
|
||||
def sample(self, logits, contexts, random_sampler=None, skip_mask=None):
|
||||
raise AssertionError("watermarker should not run for greedy requests")
|
||||
|
||||
sampler = object.__new__(GPUWatermarkSampler)
|
||||
@@ -155,3 +872,157 @@ def test_gpu_sampler_skips_watermarking_for_greedy_batch(monkeypatch):
|
||||
)
|
||||
|
||||
assert actual is expected
|
||||
|
||||
|
||||
def test_gpu_sampler_builds_speculative_contexts_from_drafts():
|
||||
sampler = object.__new__(GPUWatermarkSampler)
|
||||
sampler.watermarker = SimpleNamespace(context_width=2)
|
||||
sampler.req_states = SimpleNamespace(
|
||||
all_token_ids=SimpleNamespace(gpu=torch.tensor([[100, 101, 10, 11, 0, 0]])),
|
||||
prompt_len=SimpleNamespace(gpu=torch.tensor([2])),
|
||||
total_len=SimpleNamespace(gpu=torch.tensor([4])),
|
||||
)
|
||||
|
||||
contexts = sampler._get_contexts(
|
||||
torch.tensor([0, 0, 0]),
|
||||
torch.tensor([0, 1, 2]),
|
||||
torch.tensor([11, 20, 21]),
|
||||
)
|
||||
|
||||
assert torch.equal(contexts, torch.tensor([[10, 11], [11, 20], [20, 21]]))
|
||||
|
||||
|
||||
def test_gpu_sampler_builds_chunked_multi_request_speculative_contexts():
|
||||
sampler = object.__new__(GPUWatermarkSampler)
|
||||
sampler.watermarker = SimpleNamespace(context_width=2)
|
||||
sampler.req_states = SimpleNamespace(
|
||||
all_token_ids=SimpleNamespace(
|
||||
gpu=torch.tensor(
|
||||
[
|
||||
[100, 101, 10, 11, 0, 0],
|
||||
[200, 201, 30, 31, 0, 0],
|
||||
[300, 301, 50, 51, 0, 0],
|
||||
]
|
||||
)
|
||||
),
|
||||
prompt_len=SimpleNamespace(gpu=torch.tensor([2, 2, 2])),
|
||||
total_len=SimpleNamespace(gpu=torch.tensor([4, 4, 4])),
|
||||
)
|
||||
|
||||
contexts = sampler._get_contexts(
|
||||
torch.tensor([2, 2, 1, 1]),
|
||||
torch.tensor([0, 1, 0, 1]),
|
||||
torch.tensor([51, 60, 31, 40]),
|
||||
)
|
||||
|
||||
assert torch.equal(
|
||||
contexts,
|
||||
torch.tensor([[50, 51], [51, 60], [30, 31], [31, 40]]),
|
||||
)
|
||||
|
||||
|
||||
def test_draft_sampler_uses_draft_key_and_advances_context(monkeypatch):
|
||||
class StubSpeculator(DraftModelSpeculator):
|
||||
def capture(self): ...
|
||||
|
||||
def init_cudagraph_manager(self, cudagraph_mode): ...
|
||||
|
||||
def load_draft_model(self, target_model, target_attn_layer_names): ...
|
||||
|
||||
def propose(self, *args, **kwargs): ...
|
||||
|
||||
class StubModel:
|
||||
@staticmethod
|
||||
def compute_logits(hidden_states):
|
||||
return torch.zeros(hidden_states.shape[0], 8)
|
||||
|
||||
class StubWatermarker:
|
||||
@staticmethod
|
||||
def sample(logits, contexts, random_sampler=None):
|
||||
assert random_sampler is None
|
||||
return WatermarkSample(torch.tensor([7, 7]), logits)
|
||||
|
||||
speculator = object.__new__(StubSpeculator)
|
||||
speculator.model = StubModel()
|
||||
speculator.use_fp64_gumbel = False
|
||||
draft_watermarker = object.__new__(DraftWatermarker)
|
||||
draft_watermarker.watermarker = StubWatermarker()
|
||||
draft_watermarker.contexts = torch.tensor([[1, 2], [3, 4]])
|
||||
draft_watermarker.enabled = torch.tensor([True, False])
|
||||
speculator.draft_watermarker = draft_watermarker
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.worker.gpu.spec_decode.speculator.gumbel_sample",
|
||||
lambda *args, **kwargs: torch.tensor([3, 4]),
|
||||
)
|
||||
|
||||
sampled = speculator.sample_draft(
|
||||
hidden_states=torch.zeros(2, 4),
|
||||
sample_src_positions=torch.zeros(2, dtype=torch.int64),
|
||||
idx_mapping=torch.tensor([0, 1]),
|
||||
temperature=torch.ones(2),
|
||||
seeds=torch.zeros(2, dtype=torch.int64),
|
||||
draft_step=torch.tensor(0),
|
||||
draft_logits=torch.zeros(2, 1, 8),
|
||||
)
|
||||
|
||||
assert torch.equal(sampled, torch.tensor([7, 4]))
|
||||
assert torch.equal(draft_watermarker.contexts, torch.tensor([[2, 7], [4, 4]]))
|
||||
|
||||
|
||||
def test_dspark_reduced_vocab_draft_sampler_applies_watermarking(monkeypatch):
|
||||
speculator = object.__new__(DSparkSpeculator)
|
||||
speculator.draft_logits = torch.zeros(2, 1, 8)
|
||||
speculator._d2t_scatter_index = torch.tensor([1, 5])
|
||||
speculator._draft_scatter_buf = torch.full((2, 8), float("-inf"))
|
||||
speculator.temperature = torch.ones(2)
|
||||
speculator.seeds = torch.zeros(2, dtype=torch.int64)
|
||||
speculator._step_cols = torch.tensor([0])
|
||||
speculator.use_fp64_gumbel = False
|
||||
watermark_logits: list[torch.Tensor] = []
|
||||
|
||||
def sample(logits, sampled, idx_map, temperature):
|
||||
watermark_logits.append(logits.clone())
|
||||
return sampled + 1
|
||||
|
||||
speculator.draft_watermarker = SimpleNamespace(sample=sample)
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.worker.gpu.spec_decode.dspark.speculator.gumbel_sample",
|
||||
lambda *args, **kwargs: torch.tensor([3, 4]),
|
||||
)
|
||||
|
||||
sampled = speculator._sample_logits(
|
||||
torch.tensor([[10.0, 20.0], [30.0, 40.0]]),
|
||||
torch.tensor([0, 1]),
|
||||
torch.tensor([1, 1]),
|
||||
0,
|
||||
)
|
||||
|
||||
assert torch.equal(sampled, torch.tensor([4, 5]))
|
||||
assert torch.equal(
|
||||
watermark_logits[0][:, [1, 5]], torch.tensor([[10, 20], [30, 40]])
|
||||
)
|
||||
assert torch.isneginf(watermark_logits[0][:, [0, 2, 3, 4, 6, 7]]).all()
|
||||
|
||||
|
||||
def test_dspark_target_only_watermarking_leaves_drafts_unwatermarked(monkeypatch):
|
||||
speculator = object.__new__(DSparkSpeculator)
|
||||
speculator.draft_logits = torch.zeros(2, 1, 8)
|
||||
speculator._d2t_scatter_index = None
|
||||
speculator.temperature = torch.ones(2)
|
||||
speculator.seeds = torch.zeros(2, dtype=torch.int64)
|
||||
speculator._step_cols = torch.tensor([0])
|
||||
speculator.use_fp64_gumbel = False
|
||||
speculator.draft_watermarker = None
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.worker.gpu.spec_decode.dspark.speculator.gumbel_sample",
|
||||
lambda *args, **kwargs: torch.tensor([3, 4]),
|
||||
)
|
||||
|
||||
sampled = speculator._sample_logits(
|
||||
torch.zeros(2, 8),
|
||||
torch.tensor([0, 1]),
|
||||
torch.tensor([1, 1]),
|
||||
0,
|
||||
)
|
||||
|
||||
assert torch.equal(sampled, torch.tensor([3, 4]))
|
||||
|
||||
Reference in New Issue
Block a user