[watermarking] Dual-key gumbel-max watermarking for speculative decoding support (#56122)
Signed-off-by: Raphael Rialland <raphael.rialland@mistral.ai> Signed-off-by: Simon Veitner <sveitner@redhat.com> Signed-off-by: Tomas Ruiz <tomas.ruiz.te@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Simon Veitner <sveitner@redhat.com> Co-authored-by: Tomas Ruiz <tomas.ruiz.te@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
Simon Veitner
Tomas Ruiz
parent
1ee4be4dbc
commit
7ee8a6dd01
+180
-5
@@ -2856,14 +2856,183 @@ def _watermarked_vllm_config() -> VllmConfig:
|
||||
return config
|
||||
|
||||
|
||||
def test_gumbel_watermark_rejects_speculative_decoding():
|
||||
def test_target_only_gumbel_allows_speculative_decoding(caplog_vllm, disable_log_dedup):
|
||||
config = _watermarked_vllm_config()
|
||||
config.speculative_config = SpeculativeConfig(
|
||||
method="ngram",
|
||||
num_speculative_tokens=1,
|
||||
config.watermark_config = WatermarkConfig(
|
||||
key=42, allow_target_only_watermarking=True
|
||||
)
|
||||
config.speculative_config = SimpleNamespace(
|
||||
method="mtp",
|
||||
draft_sample_method="probabilistic",
|
||||
rejection_sample_method="standard",
|
||||
parallel_drafting=False,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="does not support speculative decoding"):
|
||||
with caplog_vllm.at_level(logging.WARNING):
|
||||
config._check_watermarking_unsupported()
|
||||
|
||||
assert "Target-only watermarking leaves accepted draft tokens" in caplog_vllm.text
|
||||
assert "Context deduplication is not supported" in caplog_vllm.text
|
||||
|
||||
|
||||
def test_speculative_watermarking_without_context_dedup_does_not_warn(
|
||||
caplog_vllm, disable_log_dedup
|
||||
):
|
||||
config = _watermarked_vllm_config()
|
||||
config.watermark_config = WatermarkConfig(
|
||||
algorithm="dual_key_gumbel", key=42, deduplicate_contexts="none"
|
||||
)
|
||||
config.speculative_config = SimpleNamespace(
|
||||
method="mtp",
|
||||
draft_sample_method="probabilistic",
|
||||
rejection_sample_method="standard",
|
||||
parallel_drafting=False,
|
||||
)
|
||||
|
||||
with caplog_vllm.at_level(logging.WARNING):
|
||||
config._check_watermarking_unsupported()
|
||||
|
||||
assert "Context deduplication is not supported" not in caplog_vllm.text
|
||||
|
||||
|
||||
def test_gumbel_rejects_speculative_decoding_without_target_only():
|
||||
config = _watermarked_vllm_config()
|
||||
config.watermark_config = WatermarkConfig(
|
||||
algorithm="gumbel", key=42, allow_target_only_watermarking=False
|
||||
)
|
||||
config.speculative_config = SimpleNamespace(
|
||||
method="mtp",
|
||||
draft_sample_method="probabilistic",
|
||||
rejection_sample_method="standard",
|
||||
parallel_drafting=False,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="'gumbel'.*allow_target_only_watermarking"):
|
||||
config._check_watermarking_unsupported()
|
||||
|
||||
|
||||
def test_dual_key_gumbel_warns_that_configured_alpha_is_unused(
|
||||
caplog_vllm, disable_log_dedup
|
||||
):
|
||||
config = _watermarked_vllm_config()
|
||||
config.watermark_config = WatermarkConfig(
|
||||
algorithm="dual_key_gumbel", key=42, alpha=0.25
|
||||
)
|
||||
config.speculative_config = SimpleNamespace(
|
||||
method="mtp",
|
||||
draft_sample_method="probabilistic",
|
||||
rejection_sample_method="standard",
|
||||
parallel_drafting=False,
|
||||
)
|
||||
|
||||
with caplog_vllm.at_level(logging.WARNING):
|
||||
config._check_watermarking_unsupported()
|
||||
|
||||
assert "The configured alpha=0.25 is not used" in caplog_vllm.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("alpha", "speculative"),
|
||||
[(0.1, True), (0.25, False)],
|
||||
ids=["default-alpha", "no-specdec"],
|
||||
)
|
||||
def test_dual_key_gumbel_alpha_warning_is_not_emitted(
|
||||
caplog_vllm, disable_log_dedup, alpha, speculative
|
||||
):
|
||||
config = _watermarked_vllm_config()
|
||||
config.watermark_config = WatermarkConfig(
|
||||
algorithm="dual_key_gumbel", key=42, alpha=alpha
|
||||
)
|
||||
if speculative:
|
||||
config.speculative_config = SimpleNamespace(
|
||||
method="mtp",
|
||||
draft_sample_method="probabilistic",
|
||||
rejection_sample_method="standard",
|
||||
parallel_drafting=False,
|
||||
)
|
||||
|
||||
with caplog_vllm.at_level(logging.WARNING):
|
||||
config._check_watermarking_unsupported()
|
||||
|
||||
assert "is not used" not in caplog_vllm.text
|
||||
|
||||
|
||||
def test_dual_key_gumbel_requires_probabilistic_drafting():
|
||||
config = _watermarked_vllm_config()
|
||||
config.watermark_config = WatermarkConfig(algorithm="dual_key_gumbel", key=42)
|
||||
config.speculative_config = SimpleNamespace(
|
||||
method="mtp",
|
||||
draft_sample_method="greedy",
|
||||
rejection_sample_method="standard",
|
||||
parallel_drafting=False,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="draft_sample_method='probabilistic'"):
|
||||
config._check_watermarking_unsupported()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method", ["eagle", "eagle3", "mtp"])
|
||||
def test_dual_key_gumbel_supports_probabilistic_speculative_decoding(method):
|
||||
config = _watermarked_vllm_config()
|
||||
config.watermark_config = WatermarkConfig(algorithm="dual_key_gumbel", key=42)
|
||||
config.speculative_config = SimpleNamespace(
|
||||
method=method,
|
||||
draft_sample_method="probabilistic",
|
||||
rejection_sample_method="standard",
|
||||
parallel_drafting=False,
|
||||
)
|
||||
|
||||
config._check_watermarking_unsupported()
|
||||
|
||||
|
||||
def test_dual_key_gumbel_supports_dspark():
|
||||
config = _watermarked_vllm_config()
|
||||
config.watermark_config = WatermarkConfig(algorithm="dual_key_gumbel", key=42)
|
||||
config.speculative_config = SimpleNamespace(
|
||||
method="dspark",
|
||||
draft_sample_method="probabilistic",
|
||||
rejection_sample_method="standard",
|
||||
parallel_drafting=True,
|
||||
)
|
||||
|
||||
config._check_watermarking_unsupported()
|
||||
|
||||
|
||||
def test_dual_key_gumbel_rejects_non_autoregressive_speculation():
|
||||
config = _watermarked_vllm_config()
|
||||
config.watermark_config = WatermarkConfig(algorithm="dual_key_gumbel", key=42)
|
||||
config.speculative_config = SimpleNamespace(
|
||||
method="ngram",
|
||||
draft_sample_method="probabilistic",
|
||||
rejection_sample_method="standard",
|
||||
parallel_drafting=False,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="autoregressive model-based"):
|
||||
config._check_watermarking_unsupported()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("overrides", "match"),
|
||||
[
|
||||
({"rejection_sample_method": "synthetic"}, "rejection_sample_method"),
|
||||
({"rejection_sample_method": "block"}, "rejection_sample_method"),
|
||||
({"parallel_drafting": True}, "Parallel speculative drafting"),
|
||||
],
|
||||
)
|
||||
def test_dual_key_gumbel_rejects_incompatible_speculative_modes(overrides, match):
|
||||
config = _watermarked_vllm_config()
|
||||
config.watermark_config = WatermarkConfig(algorithm="dual_key_gumbel", key=42)
|
||||
values = {
|
||||
"method": "mtp",
|
||||
"draft_sample_method": "probabilistic",
|
||||
"rejection_sample_method": "standard",
|
||||
"parallel_drafting": False,
|
||||
**overrides,
|
||||
}
|
||||
config.speculative_config = SimpleNamespace(**values)
|
||||
|
||||
with pytest.raises(ValueError, match=match):
|
||||
config._check_watermarking_unsupported()
|
||||
|
||||
|
||||
@@ -2882,6 +3051,12 @@ def test_watermark_key_must_fit_in_64_bits():
|
||||
WatermarkConfig(key=2**64)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("alpha", [-0.1, 1.1])
|
||||
def test_watermark_alpha_must_be_a_probability(alpha):
|
||||
with pytest.raises(ValidationError):
|
||||
WatermarkConfig(key=42, algorithm="dual_key_gumbel", alpha=alpha)
|
||||
|
||||
|
||||
def test_unknown_watermark_prf_is_rejected():
|
||||
with pytest.raises(ValidationError):
|
||||
pydantic.TypeAdapter(WatermarkConfig).validate_python(
|
||||
|
||||
Reference in New Issue
Block a user