diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index 0aede333e3..e71102ebf1 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -211,6 +211,7 @@ steps: - vllm/utils/ - vllm/v1/ - tests/v1 + - tests/watermarking device: cpu-small commands: # split the test to avoid interference @@ -222,6 +223,7 @@ steps: - pytest -v -s -m 'cpu_test' v1/kv_connector/unit - pytest -v -s -m 'cpu_test' v1/ec_connector/unit - pytest -v -s -m 'cpu_test' v1/metrics + - pytest -v -s watermarking - label: ":nvidia: (H200 MIG 18GB) Extract Hidden States Integration" key: extract-hidden-states-integration diff --git a/tests/watermarking/__init__.py b/tests/watermarking/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/tests/watermarking/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/tests/watermarking/generate_goldens.py b/tests/watermarking/generate_goldens.py new file mode 100644 index 0000000000..3cf5df1d42 --- /dev/null +++ b/tests/watermarking/generate_goldens.py @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Regenerate tests/watermarking/watermarking_goldens.json.""" + +import json +from pathlib import Path + +from tests.watermarking.golden_candidates import golden_payload, validate_golden_guards + +GOLDENS_PATH = Path(__file__).with_name("watermarking_goldens.json") + + +def main() -> None: + payload = golden_payload() + validate_golden_guards(payload["candidates"]) + GOLDENS_PATH.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + print(f"wrote {len(payload['candidates'])} candidates to {GOLDENS_PATH}") + + +if __name__ == "__main__": + main() diff --git a/tests/watermarking/golden_candidates.py b/tests/watermarking/golden_candidates.py new file mode 100644 index 0000000000..a06ef00c84 --- /dev/null +++ b/tests/watermarking/golden_candidates.py @@ -0,0 +1,885 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Deterministic watermarking golden candidates.""" + +import json +import math +from collections.abc import Mapping +from dataclasses import dataclass, replace +from pathlib import Path +from typing import get_args + +import torch + +from vllm.config.watermarking import ( + WatermarkConfig, + WatermarkContextScope, + WatermarkingAlgorithm, + WatermarkPRFName, + derive_watermark_key, +) +from vllm.v1.watermarking import ( + DualKeyGumbelWatermarkDetector, + GumbelWatermarkDetector, + WatermarkDetection, + WatermarkDetector, + create_prf, + create_watermarker, +) +from vllm.v1.worker.gpu.sample.watermark import repeated_context_mask + +GOLDEN_SCHEMA_VERSION = 3 + +# JSON floats round-trip exactly, but libm differs by an ULP across builds. +GOLDEN_FLOAT_RTOL = 1e-9 +GOLDEN_FLOAT_ATOL = 1e-300 + +P_VALUE_RATIO_MIN_DISTANCE = 1e-6 +REPETITIVE_MIN_KEY_PERTURBATION_TOKENS = 8 +REPETITIVE_MAX_P_VALUE_RATIO = 0.5 +ROUTING_BOUNDARY_MIN_DISTANCE = 1e-6 + +WATERMARK_CONFIG_FIELDS = ( + "algorithm", + "alpha", + "context_width", + "deduplicate_contexts", + "deduplicate_contexts_max_history", + "prf", + "allow_target_only_watermarking", +) + +# round(((i + 1) * (5**0.5 - 1) / 2) % 1.0, 4) for i in range(64) +ROUTING_UNIFORMS: tuple[float, ...] = ( + 0.618, 0.2361, 0.8541, 0.4721, 0.0902, 0.7082, 0.3262, 0.9443, + 0.5623, 0.1803, 0.7984, 0.4164, 0.0344, 0.6525, 0.2705, 0.8885, + 0.5066, 0.1246, 0.7426, 0.3607, 0.9787, 0.5967, 0.2148, 0.8328, + 0.4508, 0.0689, 0.6869, 0.305, 0.923, 0.541, 0.1591, 0.7771, + 0.3951, 0.0132, 0.6312, 0.2492, 0.8673, 0.4853, 0.1033, 0.7214, + 0.3394, 0.9574, 0.5755, 0.1935, 0.8115, 0.4296, 0.0476, 0.6656, + 0.2837, 0.9017, 0.5197, 0.1378, 0.7558, 0.3738, 0.9919, 0.6099, + 0.2279, 0.846, 0.464, 0.082, 0.7001, 0.3181, 0.9361, 0.5542, +) # fmt: skip + + +class GoldenFormatError(ValueError): + """The stored goldens do not have the shape this module expects.""" + + +class GoldenGuardError(ValueError): + """A candidate is too weak or too unstable to be frozen as a golden.""" + + +@dataclass(frozen=True) +class DeterministicGenerationFixture: + vocabulary_size: int = 128 + num_tokens: int = 64 + logit_denominator: int = 16 + logit_modulus: int = 131 + dominant_token: int | None = None + dominant_bias: int = 0 + prompt: tuple[int, ...] = () + routing_uniforms: tuple[float, ...] = ROUTING_UNIFORMS + + def logits(self, position: int) -> torch.Tensor: + token_ids = torch.arange(self.vocabulary_size, dtype=torch.int64) + values = (token_ids * 37 + position * 17) % self.logit_modulus - 56 + if self.dominant_token is not None: + values[self.dominant_token] += self.dominant_bias + return (values.to(torch.float32) / self.logit_denominator).unsqueeze(0) + + +@dataclass(frozen=True) +class WatermarkingSchemeConfig: + context_width: int = 4 + generation_alpha: float = 0.1 + detection_alpha: float = 0.1 + generation_deduplicate_contexts: WatermarkContextScope = "single_turn" + generation_deduplicate_contexts_max_history: int | None = 8192 + detection_deduplicate_contexts: bool = True + p_value_threshold: float = 0.01 + + +@dataclass(frozen=True) +class GenerationTrace: + routing_draws: int + key_b_routed: int | None + key_b_effective: int | None + dedup_skips: int + partial_context_skips: int + + +class _DeterministicSampler: + def __init__(self, routing_uniforms: tuple[float, ...]) -> None: + self.routing_uniforms = routing_uniforms + self.routing_position = 0 + self.key_b_routed = 0 + + def __call__(self, logits: torch.Tensor) -> torch.Tensor: + if logits.shape[-1] != 2: + return logits.argmax(dim=-1) + + probability_a = logits.softmax(dim=-1)[:, 0] + uniform = self.routing_uniforms[ + self.routing_position % len(self.routing_uniforms) + ] + self.routing_position += 1 + routed = (probability_a <= uniform).to(torch.int64) + self.key_b_routed += int(routed.sum()) + return routed + + +def routing_boundary(alpha: float) -> float: + """Return the realised key-A probability the deterministic sampler sees.""" + logits = torch.tensor([1 - alpha, alpha], dtype=torch.float32).log() + return float(logits.softmax(dim=-1)[0]) + + +def _create_gumbel_detector( + key: int, + config: WatermarkingSchemeConfig, + prf: WatermarkPRFName, +) -> WatermarkDetector: + return GumbelWatermarkDetector( + key=key, + context_width=config.context_width, + p_value_threshold=config.p_value_threshold, + prf=prf, + deduplicate_contexts=config.detection_deduplicate_contexts, + ) + + +def _create_dual_key_gumbel_detector( + key: int, + config: WatermarkingSchemeConfig, + prf: WatermarkPRFName, +) -> WatermarkDetector: + return DualKeyGumbelWatermarkDetector( + key=key, + context_width=config.context_width, + p_value_threshold=config.p_value_threshold, + prf=prf, + deduplicate_contexts=config.detection_deduplicate_contexts, + alpha=config.detection_alpha, + ) + + +DETECTOR_FACTORIES = { + "gumbel": _create_gumbel_detector, + "dual_key_gumbel": _create_dual_key_gumbel_detector, +} + + +@dataclass(frozen=True) +class WatermarkingCandidate: + id: str + scheme: WatermarkingAlgorithm + scheme_config: WatermarkingSchemeConfig + prf: WatermarkPRFName + key: int + detection_key: int + fixture: DeterministicGenerationFixture = DeterministicGenerationFixture() + + def configuration(self) -> dict[str, object]: + prf_version = getattr(create_prf(self.prf, self.key), "version", None) + if not isinstance(prf_version, str): + raise ValueError(f"PRF {self.prf} does not define a version") + return { + "scheme": self.scheme, + "scheme_config": { + "context_width": self.scheme_config.context_width, + "generation_alpha": self.scheme_config.generation_alpha, + "detection_alpha": self.scheme_config.detection_alpha, + "generation_deduplicate_contexts": ( + self.scheme_config.generation_deduplicate_contexts + ), + "generation_deduplicate_contexts_max_history": ( + self.scheme_config.generation_deduplicate_contexts_max_history + ), + "detection_deduplicate_contexts": ( + self.scheme_config.detection_deduplicate_contexts + ), + "p_value_threshold": self.scheme_config.p_value_threshold, + }, + "prf": self.prf, + "prf_version": prf_version, + "key": str(self.key), + "detection_key": str(self.detection_key), + "fixture": { + "vocabulary_size": self.fixture.vocabulary_size, + "num_tokens": self.fixture.num_tokens, + "logit_denominator": self.fixture.logit_denominator, + "logit_modulus": self.fixture.logit_modulus, + "dominant_token": self.fixture.dominant_token, + "dominant_bias": self.fixture.dominant_bias, + "prompt": list(self.fixture.prompt), + }, + } + + def resolved(self) -> dict[str, object]: + """Return the vLLM-side state this candidate resolves to. + + An allowlist rather than the whole dataclass, so a new production field + is a deliberate golden change: WATERMARK_CONFIG_FIELDS names the fields + recorded here and test_goldens.py checks it still covers + WatermarkConfig. Keys exceed 2**53 and are decimal strings. + """ + config = self._watermark_config() + detector = self._detector() + key_b_prf = getattr(detector, "key_b_prf", None) + dual_key = self.scheme == "dual_key_gumbel" + return { + "watermark_config": { + field: getattr(config, field) for field in WATERMARK_CONFIG_FIELDS + }, + "derived_keys": { + "key_a": str(derive_watermark_key(self.key, b"key_a")) + if dual_key + else None, + "key_b": str(derive_watermark_key(self.key, b"key_b")) + if dual_key + else None, + }, + "detector": { + "type": type(detector).__name__, + "context_width": detector.context_width, + "p_value_threshold": detector.p_value_threshold, + "deduplicate_contexts": detector.deduplicate_contexts, + "alpha": getattr(detector, "alpha", None), + "prf_key": str(detector.prf.key), + "key_b_prf_key": None if key_b_prf is None else str(key_b_prf.key), + }, + } + + def generate(self) -> list[int]: + return self.generate_with_trace()[0] + + def generate_with_trace(self) -> tuple[list[int], GenerationTrace]: + watermarker = create_watermarker(self._watermark_config()) + sampler = _DeterministicSampler(self.fixture.routing_uniforms) + generated: list[int] = [] + dedup_skips = 0 + partial_context_skips = 0 + key_b_effective = 0 + + for position in range(self.fixture.num_tokens): + context = self._context(generated) + skip_mask = self._skip_mask(generated, context) + skipped = skip_mask is not None and bool(skip_mask.item()) + if skipped: + dedup_skips += 1 + if len(generated) < self.scheme_config.context_width: + partial_context_skips += 1 + routed_before = sampler.key_b_routed + sample = watermarker.sample( + self.fixture.logits(position), + context, + sampler, + skip_mask, + ) + if sampler.key_b_routed > routed_before and not skipped: + key_b_effective += 1 + generated.append(int(sample.token_ids.item())) + + routes = self.scheme == "dual_key_gumbel" and ( + 0 < self.scheme_config.generation_alpha < 1 + ) + trace = GenerationTrace( + routing_draws=sampler.routing_position, + key_b_routed=sampler.key_b_routed if routes else None, + key_b_effective=key_b_effective if routes else None, + dedup_skips=dedup_skips, + partial_context_skips=partial_context_skips, + ) + return generated, trace + + def detect(self, token_ids: list[int]) -> WatermarkDetection: + return self._detector().detect(token_ids) + + def golden_entry( + self, + detection_token_ids: list[int] | None = None, + ) -> dict: + """Generate, detect and trace this candidate into one golden entry. + + ``detection_token_ids`` runs detection over stored tokens instead of + the freshly generated ones, so a token drift and a detection drift are + reported independently. + """ + token_ids, trace = self.generate_with_trace() + scored = token_ids if detection_token_ids is None else detection_token_ids + detection = self.detect(scored) + threshold = self.scheme_config.p_value_threshold + return { + "configuration": self.configuration(), + "resolved": self.resolved(), + "generation": token_ids, + "trace": _trace_payload(trace), + "detection": { + "score": detection.score, + "p_value": detection.p_value, + "p_value_ratio": detection.p_value / threshold, + "num_scored_tokens": detection.num_scored_tokens, + "is_watermarked": detection.is_watermarked, + }, + } + + def _watermark_config(self) -> WatermarkConfig: + return WatermarkConfig( + algorithm=self.scheme, + key=self.key, + alpha=self.scheme_config.generation_alpha, + context_width=self.scheme_config.context_width, + deduplicate_contexts=(self.scheme_config.generation_deduplicate_contexts), + deduplicate_contexts_max_history=( + self.scheme_config.generation_deduplicate_contexts_max_history + ), + prf=self.prf, + ) + + def _detector(self) -> WatermarkDetector: + return DETECTOR_FACTORIES[self.scheme]( + self.detection_key, + self.scheme_config, + self.prf, + ) + + def _context(self, generated: list[int]) -> torch.Tensor: + width = self.scheme_config.context_width + padded = [-1] * width + generated + return torch.tensor([padded[-width:]], dtype=torch.int64) + + def _skip_mask( + self, generated: list[int], context: torch.Tensor + ) -> torch.Tensor | None: + scope = self.scheme_config.generation_deduplicate_contexts + if scope == "none": + return None + + prompt_len = len(self.fixture.prompt) + storage = torch.zeros( + (1, prompt_len + self.fixture.num_tokens), + dtype=torch.int64, + ) + if prompt_len: + storage[0, :prompt_len] = torch.tensor(self.fixture.prompt) + if generated: + storage[0, prompt_len : prompt_len + len(generated)] = torch.tensor( + generated + ) + return repeated_context_mask( + storage, + torch.tensor([0]), + torch.tensor([prompt_len]), + torch.tensor([prompt_len + len(generated)]), + context, + self.scheme_config.generation_deduplicate_contexts_max_history, + include_prompt=scope == "all", + skip_partial_context=scope == "all", + ) + + +def _trace_payload(trace: GenerationTrace) -> dict: + return { + "routing_draws": trace.routing_draws, + "key_b_routed": trace.key_b_routed, + "key_b_effective": trace.key_b_effective, + "dedup_skips": trace.dedup_skips, + "partial_context_skips": trace.partial_context_skips, + } + + +BALANCED_FIXTURE = DeterministicGenerationFixture() + +# The first five tokens of BALANCED_FIXTURE under scope "all", so position 4 repeats. +PROMPT_FIXTURE = DeterministicGenerationFixture(prompt=(46, 42, 38, 34, 122)) + +REPETITIVE_FIXTURE = DeterministicGenerationFixture( + vocabulary_size=16, + num_tokens=48, + dominant_token=3, + dominant_bias=52, +) + +MIDDLE_FIXTURE = DeterministicGenerationFixture( + vocabulary_size=8, + num_tokens=64, + dominant_token=3, + dominant_bias=16, +) + +# Few enough scored tokens that the p_value lands just under its threshold. +NEAR_THRESHOLD_FIXTURE = DeterministicGenerationFixture( + vocabulary_size=16, + num_tokens=8, +) + + +def _candidate( + id: str, + scheme: WatermarkingAlgorithm, + key: int, + *, + prf: WatermarkPRFName, + detection_key: int | None = None, + context_width: int = 4, + generation_alpha: float = 0.1, + detection_alpha: float | None = None, + generation_deduplicate_contexts: WatermarkContextScope = "single_turn", + generation_deduplicate_contexts_max_history: int | None = 8192, + detection_deduplicate_contexts: bool = True, + p_value_threshold: float = 0.01, + fixture: DeterministicGenerationFixture = BALANCED_FIXTURE, +) -> WatermarkingCandidate: + return WatermarkingCandidate( + id=id, + scheme=scheme, + scheme_config=WatermarkingSchemeConfig( + context_width=context_width, + generation_alpha=generation_alpha, + detection_alpha=( + generation_alpha if detection_alpha is None else detection_alpha + ), + generation_deduplicate_contexts=generation_deduplicate_contexts, + generation_deduplicate_contexts_max_history=( + generation_deduplicate_contexts_max_history + ), + detection_deduplicate_contexts=detection_deduplicate_contexts, + p_value_threshold=p_value_threshold, + ), + prf=prf, + key=key, + detection_key=key if detection_key is None else detection_key, + fixture=fixture, + ) + + +WATERMARKING_CANDIDATES = ( + _candidate("gumbel-philox-key0-cw1", "gumbel", 0, prf="philox", context_width=1), + _candidate("gumbel-philox-key42-cw4", "gumbel", 42, prf="philox"), + _candidate( + "gumbel-philox-key42-cw4-wrong-key", + "gumbel", + 42, + prf="philox", + detection_key=43, + ), + _candidate( + "gumbel-philox-key2p32-cw5-no-dedup", + "gumbel", + 2**32, + prf="philox", + context_width=5, + generation_deduplicate_contexts="none", + detection_deduplicate_contexts=False, + ), + _candidate( + "gumbel-philox-keymax-cw16-all", + "gumbel", + 2**64 - 1, + prf="philox", + context_width=16, + generation_deduplicate_contexts="all", + ), + _candidate( + "gumbel-philox-key42-cw4-all", + "gumbel", + 42, + prf="philox", + generation_deduplicate_contexts="all", + ), + _candidate( + "gumbel-philox-key42-cw4-all-prompt", + "gumbel", + 42, + prf="philox", + generation_deduplicate_contexts="all", + fixture=PROMPT_FIXTURE, + ), + _candidate( + "gumbel-philox-key42-cw4-mid-history8", + "gumbel", + 42, + prf="philox", + generation_deduplicate_contexts_max_history=8, + fixture=MIDDLE_FIXTURE, + ), + _candidate( + "gumbel-philox-key42-cw4-mid-history-none", + "gumbel", + 42, + prf="philox", + generation_deduplicate_contexts_max_history=None, + fixture=MIDDLE_FIXTURE, + ), + _candidate( + "gumbel-philox-key42-cw4-mid-no-dedup", + "gumbel", + 42, + prf="philox", + generation_deduplicate_contexts="none", + generation_deduplicate_contexts_max_history=None, + detection_deduplicate_contexts=False, + fixture=MIDDLE_FIXTURE, + ), + _candidate( + "gumbel-philox-key42-cw4-mid-all", + "gumbel", + 42, + prf="philox", + generation_deduplicate_contexts="all", + generation_deduplicate_contexts_max_history=None, + fixture=MIDDLE_FIXTURE, + ), + _candidate( + "gumbel-philox-key42-cw4-near-threshold", + "gumbel", + 42, + prf="philox", + p_value_threshold=0.0015, + fixture=NEAR_THRESHOLD_FIXTURE, + ), + _candidate( + "gumbel-philox-repeated-cw1", + "gumbel", + 42, + prf="philox", + context_width=1, + fixture=REPETITIVE_FIXTURE, + ), + _candidate( + "gumbel-philox-repeated-cw4-all", + "gumbel", + 42, + prf="philox", + generation_deduplicate_contexts="all", + fixture=REPETITIVE_FIXTURE, + ), + _candidate( + "dual-key-gumbel-philox-key0-cw1-alpha0", + "dual_key_gumbel", + 0, + prf="philox", + context_width=1, + generation_alpha=0.0, + ), + _candidate( + "dual-key-gumbel-philox-key0-cw4-alpha0.1", + "dual_key_gumbel", + 0, + prf="philox", + ), + _candidate( + "dual-key-gumbel-philox-key42-cw4-alpha-mismatch", + "dual_key_gumbel", + 42, + prf="philox", + generation_alpha=0.1, + detection_alpha=0.2, + ), + _candidate( + "dual-key-gumbel-philox-key2p32-cw5-alpha0.5", + "dual_key_gumbel", + 2**32, + prf="philox", + context_width=5, + generation_alpha=0.5, + ), + _candidate( + "dual-key-gumbel-philox-keymax-cw16-alpha1", + "dual_key_gumbel", + 2**64 - 1, + prf="philox", + context_width=16, + generation_alpha=1.0, + ), + _candidate( + "dual-key-gumbel-philox-keymax-cw4-alpha0.4", + "dual_key_gumbel", + 2**64 - 1, + prf="philox", + generation_alpha=0.4, + ), + _candidate( + "dual-key-gumbel-philox-key7-cw4-near-threshold", + "dual_key_gumbel", + 7, + prf="philox", + p_value_threshold=0.003, + fixture=NEAR_THRESHOLD_FIXTURE, + ), + _candidate( + "dual-key-gumbel-philox-repeated-cw1", + "dual_key_gumbel", + 42, + prf="philox", + context_width=1, + fixture=REPETITIVE_FIXTURE, + ), + _candidate( + "dual-key-gumbel-philox-repeated-cw4-all", + "dual_key_gumbel", + 42, + prf="philox", + generation_deduplicate_contexts="all", + fixture=REPETITIVE_FIXTURE, + ), +) + +DEDUPLICATION_TWINS = ( + ( + "gumbel-philox-key42-cw4-mid-history-none", + "gumbel-philox-key42-cw4-mid-no-dedup", + ), +) + + +def validate_golden_guards(entries: Mapping[str, dict]) -> None: + candidates_by_id = { + candidate.id: candidate for candidate in WATERMARKING_CANDIDATES + } + combinations = { + (candidate.scheme, candidate.prf) for candidate in WATERMARKING_CANDIDATES + } + expected_combinations = configured_algorithm_prf_combinations() + if combinations != expected_combinations: + missing_combinations = sorted(expected_combinations - combinations) + unexpected_combinations = sorted(combinations - expected_combinations) + raise GoldenGuardError( + "candidate scheme/PRF coverage mismatch: " + f"missing {missing_combinations}, unexpected {unexpected_combinations}" + ) + + missing = sorted(set(candidates_by_id) - set(entries)) + if missing: + raise GoldenGuardError(f"golden entries are missing candidates: {missing}") + + for candidate in WATERMARKING_CANDIDATES: + entry = entries[candidate.id] + ratio = entry["detection"]["p_value_ratio"] + if abs(ratio - 1.0) < P_VALUE_RATIO_MIN_DISTANCE: + raise GoldenGuardError( + f"{candidate.id}: p_value is within " + f"{P_VALUE_RATIO_MIN_DISTANCE:g} of p_value_threshold " + f"(ratio {ratio!r}), so is_watermarked is not reproducible" + ) + + if candidate.fixture == REPETITIVE_FIXTURE: + alternate_tokens = replace(candidate, key=candidate.key ^ 1).generate() + changed_tokens = sum( + actual != alternate + for actual, alternate in zip( + entry["generation"], alternate_tokens, strict=True + ) + ) + if changed_tokens < REPETITIVE_MIN_KEY_PERTURBATION_TOKENS: + raise GoldenGuardError( + f"{candidate.id}: key ^ 1 changes only {changed_tokens} tokens; " + f"expected at least {REPETITIVE_MIN_KEY_PERTURBATION_TOKENS}" + ) + full_context_skips = ( + entry["trace"]["dedup_skips"] - entry["trace"]["partial_context_skips"] + ) + if full_context_skips < 1: + raise GoldenGuardError( + f"{candidate.id}: repetitive fixture has no full-context " + "deduplication skip" + ) + if ratio > REPETITIVE_MAX_P_VALUE_RATIO: + raise GoldenGuardError( + f"{candidate.id}: p_value_ratio {ratio!r} exceeds " + f"{REPETITIVE_MAX_P_VALUE_RATIO}" + ) + + alpha = candidate.scheme_config.generation_alpha + if candidate.scheme == "dual_key_gumbel" and 0 < alpha < 1: + boundary = routing_boundary(alpha) + distance = min( + abs(uniform - boundary) + for uniform in candidate.fixture.routing_uniforms + ) + if distance < ROUTING_BOUNDARY_MIN_DISTANCE: + raise GoldenGuardError( + f"{candidate.id}: routing uniform is only {distance!r} from " + f"the realised boundary; expected at least " + f"{ROUTING_BOUNDARY_MIN_DISTANCE:g}" + ) + + for enabled_id, disabled_id in DEDUPLICATION_TWINS: + enabled = candidates_by_id[enabled_id] + disabled = candidates_by_id[disabled_id] + expected_disabled = replace( + enabled, + id=disabled_id, + scheme_config=replace( + enabled.scheme_config, + generation_deduplicate_contexts="none", + detection_deduplicate_contexts=False, + ), + ) + if disabled != expected_disabled: + raise GoldenGuardError( + f"{enabled_id} and {disabled_id} must differ only in generation " + "and detection deduplication" + ) + + enabled_entry = entries[enabled_id] + disabled_entry = entries[disabled_id] + if enabled_entry["trace"]["dedup_skips"] < 1: + raise GoldenGuardError(f"{enabled_id}: expected at least one dedup skip") + if disabled_entry["trace"]["dedup_skips"] != 0: + raise GoldenGuardError(f"{disabled_id}: expected zero dedup skips") + if enabled_entry["generation"] == disabled_entry["generation"]: + raise GoldenGuardError( + f"{enabled_id} and {disabled_id}: generations must differ" + ) + enabled_scored = enabled_entry["detection"]["num_scored_tokens"] + disabled_scored = disabled_entry["detection"]["num_scored_tokens"] + if enabled_scored == disabled_scored: + raise GoldenGuardError( + f"{enabled_id} and {disabled_id}: detector scored-token counts " + "must differ" + ) + + +def golden_payload() -> dict: + candidates = { + candidate.id: candidate.golden_entry() for candidate in WATERMARKING_CANDIDATES + } + return { + "schema_version": GOLDEN_SCHEMA_VERSION, + "candidates": candidates, + } + + +def floats_match(actual: float, expected: float) -> bool: + return math.isclose( + actual, + expected, + rel_tol=GOLDEN_FLOAT_RTOL, + abs_tol=GOLDEN_FLOAT_ATOL, + ) + + +def _relative_difference(actual: float, expected: float) -> float: + if expected == 0.0: + return 0.0 if actual == 0.0 else math.inf + return abs(actual - expected) / abs(expected) + + +def _compare_float( + field: str, + actual: float, + expected: float, + differences: list[str], +) -> None: + if not floats_match(actual, expected): + differences.append( + f"{field}: expected {expected!r}, got {actual!r}, " + f"relative difference {_relative_difference(actual, expected):.3e}" + ) + + +def _compare_exact( + field: str, + actual: object, + expected: object, + differences: list[str], +) -> None: + if actual != expected: + differences.append(f"{field}: expected {expected!r}, got {actual!r}") + + +def _compare_mapping( + field: str, + actual: Mapping[str, object], + expected: Mapping[str, object], + differences: list[str], +) -> None: + for key in sorted(set(actual) | set(expected)): + if actual.get(key) != expected.get(key): + differences.append( + f"{field}.{key}: expected {expected.get(key)!r}, " + f"got {actual.get(key)!r}" + ) + + +def compare_entries(produced: dict, golden: dict) -> list[str]: + """Return one line per field that differs, empty when the two agree. + + Configuration, resolved state, tokens, trace, + num_scored_tokens and is_watermarked are compared exactly; score, p_value + and p_value_ratio are compared with GOLDEN_FLOAT_RTOL. + """ + differences: list[str] = [] + for field in ("configuration", "resolved", "trace"): + _compare_mapping(field, produced[field], golden[field], differences) + + if produced["generation"] != golden["generation"]: + first_difference = next( + ( + index + for index, (actual, expected) in enumerate( + zip(produced["generation"], golden["generation"]) + ) + if actual != expected + ), + min(len(produced["generation"]), len(golden["generation"])), + ) + differences.append( + f"generation: first differs at index {first_difference} " + f"({len(produced['generation'])} tokens generated, " + f"{len(golden['generation'])} in the golden): " + f"expected {golden['generation']}, got {produced['generation']}" + ) + + detection = produced["detection"] + expected_detection = golden["detection"] + for field in ("score", "p_value", "p_value_ratio"): + _compare_float( + f"detection.{field}", + detection[field], + expected_detection[field], + differences, + ) + for field in ("num_scored_tokens", "is_watermarked"): + _compare_exact( + f"detection.{field}", + detection[field], + expected_detection[field], + differences, + ) + return differences + + +def compare_golden(candidate: WatermarkingCandidate, golden: dict) -> list[str]: + """Return one line per field of ``candidate`` that drifted from ``golden``.""" + return compare_entries(candidate.golden_entry(golden["generation"]), golden) + + +def read_goldens(path: Path) -> dict[str, dict]: + """Read the goldens file and return its candidate mapping.""" + payload = json.loads(path.read_text(encoding="utf-8")) + version = payload.get("schema_version") + if version != GOLDEN_SCHEMA_VERSION: + raise GoldenFormatError( + f"{path}: schema_version {version!r}, expected {GOLDEN_SCHEMA_VERSION}" + ) + return payload["candidates"] + + +def configured_algorithms() -> set[str]: + return set(get_args(WatermarkingAlgorithm)) + + +def configured_prfs() -> set[str]: + return set(get_args(WatermarkPRFName)) + + +def configured_algorithm_prf_combinations() -> set[tuple[str, str]]: + return { + (algorithm, prf) + for algorithm in configured_algorithms() + for prf in configured_prfs() + } diff --git a/tests/watermarking/test_goldens.py b/tests/watermarking/test_goldens.py new file mode 100644 index 0000000000..5e12a9af88 --- /dev/null +++ b/tests/watermarking/test_goldens.py @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Check the watermarking goldens still reproduce.""" + +import dataclasses +from pathlib import Path + +import pytest + +from tests.watermarking.golden_candidates import ( + DETECTOR_FACTORIES, + GOLDEN_FLOAT_RTOL, + WATERMARK_CONFIG_FIELDS, + WATERMARKING_CANDIDATES, + compare_golden, + configured_algorithm_prf_combinations, + configured_algorithms, + read_goldens, + routing_boundary, + validate_golden_guards, +) +from vllm.config.watermarking import WatermarkConfig + +GOLDENS_PATH = Path(__file__).with_name("watermarking_goldens.json") + +CANDIDATES_BY_ID = {candidate.id: candidate for candidate in WATERMARKING_CANDIDATES} + +NEGATIVE_CANDIDATE_ID = "gumbel-philox-key42-cw4-wrong-key" + + +@pytest.fixture(scope="module") +def goldens() -> dict[str, dict]: + return read_goldens(GOLDENS_PATH) + + +def test_goldens_cover_every_candidate(goldens: dict[str, dict]): + candidate_ids = set(CANDIDATES_BY_ID) + + assert len(candidate_ids) == len(WATERMARKING_CANDIDATES) + assert set(goldens) == candidate_ids, "golden contract does not match candidates" + candidate_combinations = { + (candidate.scheme, candidate.prf) for candidate in WATERMARKING_CANDIDATES + } + assert candidate_combinations == configured_algorithm_prf_combinations() + assert set(DETECTOR_FACTORIES) == configured_algorithms() + + +def test_golden_fixture_guards(goldens: dict[str, dict]): + validate_golden_guards(goldens) + + +def test_resolved_records_every_watermark_config_field(): + assert {field.name for field in dataclasses.fields(WatermarkConfig)} == ( + set(WATERMARK_CONFIG_FIELDS) | {"key"} + ) + + +@pytest.mark.parametrize( + "candidate", + WATERMARKING_CANDIDATES, + ids=lambda candidate: candidate.id, +) +def test_watermarking_candidate_golden(candidate, goldens: dict[str, dict]): + differences = compare_golden(candidate, goldens[candidate.id]) + + assert not differences, "\n".join( + [ + f"{candidate.id}: golden drift " + f"(tolerance {GOLDEN_FLOAT_RTOL:g} on score and p_value)", + *differences, + ] + ) + + +def test_negative_row_is_not_watermarked(goldens: dict[str, dict]): + candidate = CANDIDATES_BY_ID[NEGATIVE_CANDIDATE_ID] + + assert candidate.detection_key != candidate.key + assert goldens[NEGATIVE_CANDIDATE_ID]["detection"]["is_watermarked"] is False + + +@pytest.mark.parametrize( + "candidate", + [ + candidate + for candidate in WATERMARKING_CANDIDATES + if candidate.scheme == "dual_key_gumbel" + and 0 < candidate.scheme_config.generation_alpha < 1 + ], + ids=lambda candidate: candidate.id, +) +def test_routing_sequence_resolves_alpha(candidate): + alpha = candidate.scheme_config.generation_alpha + uniforms = candidate.fixture.routing_uniforms + boundary = routing_boundary(alpha) + share = sum(1 for uniform in uniforms if boundary <= uniform) / len(uniforms) + + assert abs(share - alpha) <= 0.03 diff --git a/tests/watermarking/test_watermarking.py b/tests/watermarking/test_watermarking.py index 65ea9e93ad..232f19a9a6 100644 --- a/tests/watermarking/test_watermarking.py +++ b/tests/watermarking/test_watermarking.py @@ -276,9 +276,9 @@ def test_gpu_sampler_warns_when_watermarking_is_enabled_for_greedy(monkeypatch): "vllm.v1.watermarking.gpu_sampler.logger.warning_once", messages.append ) - sampler.add_request(0, 1, SamplingParams(temperature=0)) - sampler.add_request(0, 1, SamplingParams(temperature=1)) - sampler.add_request(0, 1, SamplingParams(temperature=0, watermarking=False)) + sampler.add_request(0, SamplingParams(temperature=0)) + sampler.add_request(0, SamplingParams(temperature=1)) + sampler.add_request(0, SamplingParams(temperature=0, watermarking=False)) assert messages == [ ( @@ -945,6 +945,7 @@ def test_draft_sampler_uses_draft_key_and_advances_context(monkeypatch): speculator = object.__new__(StubSpeculator) speculator.model = StubModel() speculator.use_fp64_gumbel = False + speculator.acceptance_estimator = None draft_watermarker = object.__new__(DraftWatermarker) draft_watermarker.watermarker = StubWatermarker() draft_watermarker.contexts = torch.tensor([[1, 2], [3, 4]]) @@ -978,6 +979,7 @@ def test_dspark_reduced_vocab_draft_sampler_applies_watermarking(monkeypatch): speculator.seeds = torch.zeros(2, dtype=torch.int64) speculator._step_cols = torch.tensor([0]) speculator.use_fp64_gumbel = False + speculator.acceptance_estimator = None watermark_logits: list[torch.Tensor] = [] def sample(logits, sampled, idx_map, temperature): @@ -1012,6 +1014,7 @@ def test_dspark_target_only_watermarking_leaves_drafts_unwatermarked(monkeypatch speculator.seeds = torch.zeros(2, dtype=torch.int64) speculator._step_cols = torch.tensor([0]) speculator.use_fp64_gumbel = False + speculator.acceptance_estimator = None speculator.draft_watermarker = None monkeypatch.setattr( "vllm.v1.worker.gpu.spec_decode.dspark.speculator.gumbel_sample", diff --git a/tests/watermarking/watermarking_goldens.json b/tests/watermarking/watermarking_goldens.json new file mode 100644 index 0000000000..ddcd142729 --- /dev/null +++ b/tests/watermarking/watermarking_goldens.json @@ -0,0 +1,2848 @@ +{ + "candidates": { + "dual-key-gumbel-philox-key0-cw1-alpha0": { + "configuration": { + "detection_key": "0", + "fixture": { + "dominant_bias": 0, + "dominant_token": null, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [], + "vocabulary_size": 128 + }, + "key": "0", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "dual_key_gumbel", + "scheme_config": { + "context_width": 1, + "detection_alpha": 0.0, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.0, + "generation_deduplicate_contexts": "single_turn", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 55, + "p_value": 2.7410983620034455e-44, + "p_value_ratio": 2.7410983620034455e-42, + "score": 229.87923167230588 + }, + "generation": [ + 81, + 91, + 84, + 122, + 37, + 118, + 11, + 3, + 116, + 56, + 66, + 37, + 44, + 40, + 85, + 31, + 102, + 73, + 126, + 16, + 40, + 93, + 11, + 85, + 81, + 77, + 41, + 12, + 118, + 61, + 57, + 53, + 38, + 111, + 115, + 5, + 1, + 99, + 113, + 28, + 87, + 55, + 83, + 51, + 47, + 57, + 124, + 31, + 116, + 112, + 30, + 43, + 89, + 110, + 60, + 10, + 9, + 58, + 114, + 125, + 68, + 78, + 106, + 56 + ], + "resolved": { + "derived_keys": { + "key_a": "15691511769010696906", + "key_b": "9736202488081271922" + }, + "detector": { + "alpha": 0.0, + "context_width": 1, + "deduplicate_contexts": true, + "key_b_prf_key": "9736202488081271922", + "p_value_threshold": 0.01, + "prf_key": "15691511769010696906", + "type": "DualKeyGumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "dual_key_gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.0, + "context_width": 1, + "deduplicate_contexts": "single_turn", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 9, + "key_b_effective": null, + "key_b_routed": null, + "partial_context_skips": 0, + "routing_draws": 0 + } + }, + "dual-key-gumbel-philox-key0-cw4-alpha0.1": { + "configuration": { + "detection_key": "0", + "fixture": { + "dominant_bias": 0, + "dominant_token": null, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [], + "vocabulary_size": 128 + }, + "key": "0", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "dual_key_gumbel", + "scheme_config": { + "context_width": 4, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "single_turn", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 64, + "p_value": 1.234132682420224e-51, + "p_value_ratio": 1.234132682420224e-49, + "score": 241.81126265648692 + }, + "generation": [ + 70, + 56, + 91, + 90, + 72, + 68, + 17, + 78, + 21, + 116, + 104, + 62, + 15, + 100, + 82, + 13, + 88, + 70, + 112, + 62, + 79, + 57, + 11, + 113, + 88, + 80, + 23, + 90, + 79, + 42, + 18, + 60, + 10, + 73, + 94, + 64, + 96, + 71, + 35, + 56, + 38, + 93, + 37, + 54, + 82, + 127, + 0, + 45, + 37, + 75, + 19, + 92, + 68, + 127, + 28, + 56, + 45, + 69, + 8, + 1, + 110, + 39, + 127, + 56 + ], + "resolved": { + "derived_keys": { + "key_a": "15691511769010696906", + "key_b": "9736202488081271922" + }, + "detector": { + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": true, + "key_b_prf_key": "9736202488081271922", + "p_value_threshold": 0.01, + "prf_key": "15691511769010696906", + "type": "DualKeyGumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "dual_key_gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": "single_turn", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 0, + "key_b_effective": 7, + "key_b_routed": 7, + "partial_context_skips": 0, + "routing_draws": 64 + } + }, + "dual-key-gumbel-philox-key2p32-cw5-alpha0.5": { + "configuration": { + "detection_key": "4294967296", + "fixture": { + "dominant_bias": 0, + "dominant_token": null, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [], + "vocabulary_size": 128 + }, + "key": "4294967296", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "dual_key_gumbel", + "scheme_config": { + "context_width": 5, + "detection_alpha": 0.5, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.5, + "generation_deduplicate_contexts": "single_turn", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 64, + "p_value": 6.733621085489817e-36, + "p_value_ratio": 6.7336210854898174e-34, + "score": 162.0665949787407 + }, + "generation": [ + 53, + 116, + 48, + 94, + 47, + 118, + 64, + 123, + 106, + 65, + 59, + 48, + 4, + 86, + 64, + 32, + 84, + 116, + 23, + 23, + 65, + 15, + 50, + 67, + 49, + 69, + 34, + 76, + 1, + 95, + 103, + 120, + 102, + 69, + 125, + 121, + 57, + 39, + 127, + 113, + 98, + 87, + 9, + 51, + 100, + 53, + 46, + 42, + 123, + 55, + 108, + 100, + 50, + 103, + 67, + 80, + 94, + 41, + 97, + 117, + 18, + 66, + 74, + 38 + ], + "resolved": { + "derived_keys": { + "key_a": "9864452095710749384", + "key_b": "10834826187781349534" + }, + "detector": { + "alpha": 0.5, + "context_width": 5, + "deduplicate_contexts": true, + "key_b_prf_key": "10834826187781349534", + "p_value_threshold": 0.01, + "prf_key": "9864452095710749384", + "type": "DualKeyGumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "dual_key_gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.5, + "context_width": 5, + "deduplicate_contexts": "single_turn", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 0, + "key_b_effective": 33, + "key_b_routed": 33, + "partial_context_skips": 0, + "routing_draws": 64 + } + }, + "dual-key-gumbel-philox-key42-cw4-alpha-mismatch": { + "configuration": { + "detection_key": "42", + "fixture": { + "dominant_bias": 0, + "dominant_token": null, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [], + "vocabulary_size": 128 + }, + "key": "42", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "dual_key_gumbel", + "scheme_config": { + "context_width": 4, + "detection_alpha": 0.2, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "single_turn", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 64, + "p_value": 1.6569954931292397e-49, + "p_value_ratio": 1.6569954931292398e-47, + "score": 215.24386344560963 + }, + "generation": [ + 46, + 56, + 6, + 122, + 30, + 100, + 121, + 67, + 63, + 116, + 13, + 65, + 90, + 47, + 25, + 88, + 120, + 31, + 2, + 37, + 40, + 34, + 92, + 28, + 45, + 126, + 94, + 90, + 18, + 36, + 74, + 102, + 10, + 13, + 101, + 51, + 29, + 29, + 46, + 81, + 63, + 2, + 29, + 78, + 121, + 89, + 124, + 109, + 123, + 34, + 30, + 93, + 4, + 110, + 28, + 34, + 105, + 41, + 5, + 48, + 81, + 120, + 60, + 70 + ], + "resolved": { + "derived_keys": { + "key_a": "37150756557148556", + "key_b": "15826636237805488045" + }, + "detector": { + "alpha": 0.2, + "context_width": 4, + "deduplicate_contexts": true, + "key_b_prf_key": "15826636237805488045", + "p_value_threshold": 0.01, + "prf_key": "37150756557148556", + "type": "DualKeyGumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "dual_key_gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": "single_turn", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 0, + "key_b_effective": 7, + "key_b_routed": 7, + "partial_context_skips": 0, + "routing_draws": 64 + } + }, + "dual-key-gumbel-philox-key7-cw4-near-threshold": { + "configuration": { + "detection_key": "7", + "fixture": { + "dominant_bias": 0, + "dominant_token": null, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 8, + "prompt": [], + "vocabulary_size": 16 + }, + "key": "7", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "dual_key_gumbel", + "scheme_config": { + "context_width": 4, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "single_turn", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.003 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 8, + "p_value": 0.002130807652336366, + "p_value_ratio": 0.7102692174454553, + "score": 17.2829982488662 + }, + "generation": [ + 7, + 3, + 13, + 2, + 5, + 1, + 4, + 14 + ], + "resolved": { + "derived_keys": { + "key_a": "3490214442843822077", + "key_b": "7720047515392181251" + }, + "detector": { + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": true, + "key_b_prf_key": "7720047515392181251", + "p_value_threshold": 0.003, + "prf_key": "3490214442843822077", + "type": "DualKeyGumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "dual_key_gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": "single_turn", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 0, + "key_b_effective": 1, + "key_b_routed": 1, + "partial_context_skips": 0, + "routing_draws": 8 + } + }, + "dual-key-gumbel-philox-keymax-cw16-alpha1": { + "configuration": { + "detection_key": "18446744073709551615", + "fixture": { + "dominant_bias": 0, + "dominant_token": null, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [], + "vocabulary_size": 128 + }, + "key": "18446744073709551615", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "dual_key_gumbel", + "scheme_config": { + "context_width": 16, + "detection_alpha": 1.0, + "detection_deduplicate_contexts": true, + "generation_alpha": 1.0, + "generation_deduplicate_contexts": "single_turn", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 64, + "p_value": 2.5582243247165326e-52, + "p_value_ratio": 2.558224324716533e-50, + "score": 270.9771333793902 + }, + "generation": [ + 45, + 3, + 73, + 41, + 47, + 40, + 21, + 85, + 80, + 70, + 34, + 48, + 58, + 47, + 96, + 28, + 56, + 77, + 101, + 30, + 97, + 103, + 96, + 53, + 87, + 37, + 34, + 113, + 33, + 25, + 103, + 113, + 102, + 6, + 101, + 37, + 29, + 75, + 46, + 67, + 105, + 48, + 115, + 51, + 114, + 4, + 0, + 81, + 98, + 35, + 115, + 111, + 7, + 78, + 21, + 52, + 112, + 48, + 19, + 47, + 68, + 17, + 106, + 73 + ], + "resolved": { + "derived_keys": { + "key_a": "11075965703501209888", + "key_b": "7399018451426338514" + }, + "detector": { + "alpha": 1.0, + "context_width": 16, + "deduplicate_contexts": true, + "key_b_prf_key": "7399018451426338514", + "p_value_threshold": 0.01, + "prf_key": "11075965703501209888", + "type": "DualKeyGumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "dual_key_gumbel", + "allow_target_only_watermarking": false, + "alpha": 1.0, + "context_width": 16, + "deduplicate_contexts": "single_turn", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 0, + "key_b_effective": null, + "key_b_routed": null, + "partial_context_skips": 0, + "routing_draws": 0 + } + }, + "dual-key-gumbel-philox-keymax-cw4-alpha0.4": { + "configuration": { + "detection_key": "18446744073709551615", + "fixture": { + "dominant_bias": 0, + "dominant_token": null, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [], + "vocabulary_size": 128 + }, + "key": "18446744073709551615", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "dual_key_gumbel", + "scheme_config": { + "context_width": 4, + "detection_alpha": 0.4, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.4, + "generation_deduplicate_contexts": "single_turn", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 64, + "p_value": 5.9713149418801004e-43, + "p_value_ratio": 5.9713149418801e-41, + "score": 177.98266424643373 + }, + "generation": [ + 7, + 42, + 13, + 55, + 103, + 125, + 96, + 85, + 119, + 38, + 101, + 108, + 72, + 8, + 96, + 120, + 116, + 109, + 119, + 62, + 26, + 107, + 63, + 113, + 102, + 34, + 83, + 90, + 121, + 53, + 18, + 67, + 20, + 12, + 101, + 40, + 86, + 89, + 59, + 81, + 24, + 20, + 72, + 51, + 22, + 117, + 14, + 88, + 116, + 44, + 100, + 118, + 75, + 7, + 10, + 70, + 91, + 5, + 33, + 22, + 4, + 71, + 31, + 121 + ], + "resolved": { + "derived_keys": { + "key_a": "11075965703501209888", + "key_b": "7399018451426338514" + }, + "detector": { + "alpha": 0.4, + "context_width": 4, + "deduplicate_contexts": true, + "key_b_prf_key": "7399018451426338514", + "p_value_threshold": 0.01, + "prf_key": "11075965703501209888", + "type": "DualKeyGumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "dual_key_gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.4, + "context_width": 4, + "deduplicate_contexts": "single_turn", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 0, + "key_b_effective": 26, + "key_b_routed": 26, + "partial_context_skips": 0, + "routing_draws": 64 + } + }, + "dual-key-gumbel-philox-repeated-cw1": { + "configuration": { + "detection_key": "42", + "fixture": { + "dominant_bias": 52, + "dominant_token": 3, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 48, + "prompt": [], + "vocabulary_size": 16 + }, + "key": "42", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "dual_key_gumbel", + "scheme_config": { + "context_width": 1, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "single_turn", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 13, + "p_value": 0.002750266035191538, + "p_value_ratio": 0.2750266035191538, + "score": 23.896456918277963 + }, + "generation": [ + 3, + 3, + 6, + 9, + 12, + 3, + 3, + 3, + 3, + 10, + 13, + 2, + 1, + 1, + 3, + 3, + 3, + 6, + 2, + 5, + 12, + 8, + 4, + 14, + 3, + 6, + 2, + 5, + 1, + 3, + 3, + 3, + 10, + 6, + 2, + 5, + 1, + 3, + 3, + 3, + 6, + 13, + 9, + 5, + 1, + 3, + 3, + 3 + ], + "resolved": { + "derived_keys": { + "key_a": "37150756557148556", + "key_b": "15826636237805488045" + }, + "detector": { + "alpha": 0.1, + "context_width": 1, + "deduplicate_contexts": true, + "key_b_prf_key": "15826636237805488045", + "p_value_threshold": 0.01, + "prf_key": "37150756557148556", + "type": "DualKeyGumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "dual_key_gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 1, + "deduplicate_contexts": "single_turn", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 35, + "key_b_effective": 1, + "key_b_routed": 4, + "partial_context_skips": 0, + "routing_draws": 48 + } + }, + "dual-key-gumbel-philox-repeated-cw4-all": { + "configuration": { + "detection_key": "42", + "fixture": { + "dominant_bias": 52, + "dominant_token": 3, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 48, + "prompt": [], + "vocabulary_size": 16 + }, + "key": "42", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "dual_key_gumbel", + "scheme_config": { + "context_width": 4, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "all", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 47, + "p_value": 8.656644652657153e-07, + "p_value_ratio": 8.656644652657152e-05, + "score": 82.86886159792212 + }, + "generation": [ + 3, + 3, + 6, + 2, + 3, + 1, + 11, + 0, + 14, + 10, + 5, + 15, + 5, + 3, + 7, + 7, + 3, + 15, + 9, + 12, + 11, + 3, + 14, + 3, + 3, + 6, + 2, + 5, + 8, + 11, + 3, + 3, + 10, + 13, + 2, + 12, + 1, + 3, + 3, + 3, + 13, + 13, + 9, + 12, + 3, + 3, + 3, + 3 + ], + "resolved": { + "derived_keys": { + "key_a": "37150756557148556", + "key_b": "15826636237805488045" + }, + "detector": { + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": true, + "key_b_prf_key": "15826636237805488045", + "p_value_threshold": 0.01, + "prf_key": "37150756557148556", + "type": "DualKeyGumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "dual_key_gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": "all", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 5, + "key_b_effective": 4, + "key_b_routed": 4, + "partial_context_skips": 4, + "routing_draws": 48 + } + }, + "gumbel-philox-key0-cw1": { + "configuration": { + "detection_key": "0", + "fixture": { + "dominant_bias": 0, + "dominant_token": null, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [], + "vocabulary_size": 128 + }, + "key": "0", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "gumbel", + "scheme_config": { + "context_width": 1, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "single_turn", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 54, + "p_value": 4.532248480235615e-47, + "p_value_ratio": 4.532248480235615e-45, + "score": 236.2768211235414 + }, + "generation": [ + 24, + 42, + 66, + 26, + 51, + 107, + 36, + 25, + 63, + 24, + 6, + 48, + 65, + 124, + 117, + 85, + 45, + 112, + 48, + 101, + 19, + 22, + 89, + 88, + 63, + 77, + 115, + 104, + 65, + 61, + 32, + 20, + 95, + 98, + 30, + 58, + 86, + 50, + 39, + 113, + 122, + 62, + 69, + 40, + 29, + 92, + 113, + 120, + 52, + 112, + 108, + 1, + 25, + 96, + 3, + 95, + 84, + 65, + 76, + 3, + 68, + 71, + 81, + 56 + ], + "resolved": { + "derived_keys": { + "key_a": null, + "key_b": null + }, + "detector": { + "alpha": null, + "context_width": 1, + "deduplicate_contexts": true, + "key_b_prf_key": null, + "p_value_threshold": 0.01, + "prf_key": "0", + "type": "GumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 1, + "deduplicate_contexts": "single_turn", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 10, + "key_b_effective": null, + "key_b_routed": null, + "partial_context_skips": 0, + "routing_draws": 0 + } + }, + "gumbel-philox-key2p32-cw5-no-dedup": { + "configuration": { + "detection_key": "4294967296", + "fixture": { + "dominant_bias": 0, + "dominant_token": null, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [], + "vocabulary_size": 128 + }, + "key": "4294967296", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "gumbel", + "scheme_config": { + "context_width": 5, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": false, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "none", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 64, + "p_value": 1.1027575381260964e-49, + "p_value_ratio": 1.1027575381260964e-47, + "score": 263.04926919645095 + }, + "generation": [ + 21, + 10, + 73, + 94, + 19, + 61, + 85, + 42, + 95, + 116, + 105, + 47, + 58, + 36, + 74, + 42, + 95, + 70, + 34, + 37, + 125, + 29, + 64, + 85, + 70, + 73, + 44, + 90, + 47, + 107, + 25, + 70, + 98, + 41, + 1, + 51, + 8, + 57, + 99, + 88, + 80, + 119, + 55, + 15, + 121, + 43, + 124, + 56, + 34, + 87, + 115, + 65, + 4, + 110, + 53, + 77, + 58, + 126, + 19, + 72, + 64, + 113, + 109, + 123 + ], + "resolved": { + "derived_keys": { + "key_a": null, + "key_b": null + }, + "detector": { + "alpha": null, + "context_width": 5, + "deduplicate_contexts": false, + "key_b_prf_key": null, + "p_value_threshold": 0.01, + "prf_key": "4294967296", + "type": "GumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 5, + "deduplicate_contexts": "none", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 0, + "key_b_effective": null, + "key_b_routed": null, + "partial_context_skips": 0, + "routing_draws": 0 + } + }, + "gumbel-philox-key42-cw4": { + "configuration": { + "detection_key": "42", + "fixture": { + "dominant_bias": 0, + "dominant_token": null, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [], + "vocabulary_size": 128 + }, + "key": "42", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "gumbel", + "scheme_config": { + "context_width": 4, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "single_turn", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 64, + "p_value": 7.128196280874065e-61, + "p_value_ratio": 7.128196280874065e-59, + "score": 296.27295787856264 + }, + "generation": [ + 46, + 24, + 45, + 80, + 14, + 26, + 22, + 124, + 21, + 45, + 16, + 30, + 5, + 89, + 0, + 10, + 27, + 70, + 80, + 62, + 1, + 11, + 117, + 53, + 10, + 45, + 94, + 37, + 43, + 22, + 78, + 49, + 119, + 9, + 22, + 118, + 82, + 75, + 32, + 24, + 84, + 27, + 122, + 66, + 78, + 64, + 124, + 17, + 91, + 12, + 51, + 15, + 75, + 64, + 127, + 45, + 112, + 15, + 83, + 114, + 121, + 109, + 24, + 77 + ], + "resolved": { + "derived_keys": { + "key_a": null, + "key_b": null + }, + "detector": { + "alpha": null, + "context_width": 4, + "deduplicate_contexts": true, + "key_b_prf_key": null, + "p_value_threshold": 0.01, + "prf_key": "42", + "type": "GumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": "single_turn", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 0, + "key_b_effective": null, + "key_b_routed": null, + "partial_context_skips": 0, + "routing_draws": 0 + } + }, + "gumbel-philox-key42-cw4-all": { + "configuration": { + "detection_key": "42", + "fixture": { + "dominant_bias": 0, + "dominant_token": null, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [], + "vocabulary_size": 128 + }, + "key": "42", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "gumbel", + "scheme_config": { + "context_width": 4, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "all", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 64, + "p_value": 6.0283750962334736e-55, + "p_value_ratio": 6.028375096233473e-53, + "score": 278.8159437407263 + }, + "generation": [ + 46, + 42, + 38, + 34, + 122, + 33, + 43, + 117, + 24, + 52, + 29, + 115, + 12, + 32, + 121, + 124, + 81, + 45, + 112, + 16, + 1, + 100, + 11, + 46, + 95, + 77, + 80, + 5, + 53, + 114, + 64, + 67, + 37, + 52, + 62, + 125, + 107, + 75, + 39, + 28, + 126, + 20, + 23, + 61, + 54, + 3, + 116, + 10, + 59, + 27, + 58, + 104, + 124, + 2, + 28, + 38, + 91, + 101, + 118, + 103, + 46, + 110, + 70, + 45 + ], + "resolved": { + "derived_keys": { + "key_a": null, + "key_b": null + }, + "detector": { + "alpha": null, + "context_width": 4, + "deduplicate_contexts": true, + "key_b_prf_key": null, + "p_value_threshold": 0.01, + "prf_key": "42", + "type": "GumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": "all", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 4, + "key_b_effective": null, + "key_b_routed": null, + "partial_context_skips": 4, + "routing_draws": 0 + } + }, + "gumbel-philox-key42-cw4-all-prompt": { + "configuration": { + "detection_key": "42", + "fixture": { + "dominant_bias": 0, + "dominant_token": null, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [ + 46, + 42, + 38, + 34, + 122 + ], + "vocabulary_size": 128 + }, + "key": "42", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "gumbel", + "scheme_config": { + "context_width": 4, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "all", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 64, + "p_value": 4.330336079473992e-56, + "p_value_ratio": 4.3303360794739913e-54, + "score": 282.2076506349452 + }, + "generation": [ + 46, + 42, + 38, + 34, + 30, + 43, + 121, + 123, + 91, + 17, + 66, + 58, + 54, + 103, + 28, + 39, + 3, + 34, + 58, + 23, + 21, + 29, + 57, + 9, + 66, + 91, + 101, + 58, + 85, + 67, + 18, + 102, + 123, + 112, + 2, + 37, + 100, + 32, + 35, + 67, + 116, + 115, + 108, + 5, + 100, + 14, + 67, + 56, + 122, + 23, + 12, + 125, + 103, + 20, + 70, + 8, + 45, + 1, + 90, + 45, + 114, + 10, + 30, + 66 + ], + "resolved": { + "derived_keys": { + "key_a": null, + "key_b": null + }, + "detector": { + "alpha": null, + "context_width": 4, + "deduplicate_contexts": true, + "key_b_prf_key": null, + "p_value_threshold": 0.01, + "prf_key": "42", + "type": "GumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": "all", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 5, + "key_b_effective": null, + "key_b_routed": null, + "partial_context_skips": 4, + "routing_draws": 0 + } + }, + "gumbel-philox-key42-cw4-mid-all": { + "configuration": { + "detection_key": "42", + "fixture": { + "dominant_bias": 16, + "dominant_token": 3, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [], + "vocabulary_size": 8 + }, + "key": "42", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "gumbel", + "scheme_config": { + "context_width": 4, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "all", + "generation_deduplicate_contexts_max_history": null, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 58, + "p_value": 1.6692708568346358e-08, + "p_value_ratio": 1.6692708568346358e-06, + "score": 110.3077257312174 + }, + "generation": [ + 7, + 3, + 6, + 2, + 5, + 1, + 7, + 3, + 6, + 6, + 5, + 2, + 5, + 4, + 3, + 0, + 3, + 6, + 5, + 5, + 1, + 7, + 4, + 3, + 3, + 6, + 1, + 5, + 1, + 3, + 3, + 3, + 6, + 6, + 2, + 3, + 1, + 4, + 7, + 3, + 6, + 2, + 5, + 5, + 0, + 0, + 7, + 3, + 6, + 2, + 5, + 1, + 4, + 2, + 3, + 3, + 2, + 5, + 4, + 1, + 4, + 7, + 3, + 6 + ], + "resolved": { + "derived_keys": { + "key_a": null, + "key_b": null + }, + "detector": { + "alpha": null, + "context_width": 4, + "deduplicate_contexts": true, + "key_b_prf_key": null, + "p_value_threshold": 0.01, + "prf_key": "42", + "type": "GumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": "all", + "deduplicate_contexts_max_history": null, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 10, + "key_b_effective": null, + "key_b_routed": null, + "partial_context_skips": 4, + "routing_draws": 0 + } + }, + "gumbel-philox-key42-cw4-mid-history-none": { + "configuration": { + "detection_key": "42", + "fixture": { + "dominant_bias": 16, + "dominant_token": 3, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [], + "vocabulary_size": 8 + }, + "key": "42", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "gumbel", + "scheme_config": { + "context_width": 4, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "single_turn", + "generation_deduplicate_contexts_max_history": null, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 58, + "p_value": 1.3019898400616044e-08, + "p_value_ratio": 1.3019898400616043e-06, + "score": 110.81134114351678 + }, + "generation": [ + 7, + 2, + 2, + 5, + 5, + 4, + 4, + 0, + 3, + 2, + 6, + 2, + 5, + 1, + 7, + 3, + 6, + 6, + 5, + 5, + 7, + 0, + 3, + 0, + 3, + 6, + 5, + 5, + 1, + 7, + 7, + 3, + 6, + 2, + 2, + 5, + 1, + 4, + 2, + 3, + 5, + 2, + 5, + 4, + 3, + 4, + 0, + 6, + 6, + 5, + 5, + 1, + 4, + 7, + 7, + 3, + 6, + 2, + 5, + 1, + 4, + 0, + 3, + 6 + ], + "resolved": { + "derived_keys": { + "key_a": null, + "key_b": null + }, + "detector": { + "alpha": null, + "context_width": 4, + "deduplicate_contexts": true, + "key_b_prf_key": null, + "p_value_threshold": 0.01, + "prf_key": "42", + "type": "GumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": "single_turn", + "deduplicate_contexts_max_history": null, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 6, + "key_b_effective": null, + "key_b_routed": null, + "partial_context_skips": 0, + "routing_draws": 0 + } + }, + "gumbel-philox-key42-cw4-mid-history8": { + "configuration": { + "detection_key": "42", + "fixture": { + "dominant_bias": 16, + "dominant_token": 3, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [], + "vocabulary_size": 8 + }, + "key": "42", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "gumbel", + "scheme_config": { + "context_width": 4, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "single_turn", + "generation_deduplicate_contexts_max_history": 8, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 58, + "p_value": 2.772032243308163e-09, + "p_value_ratio": 2.772032243308163e-07, + "score": 113.89758610123921 + }, + "generation": [ + 7, + 2, + 2, + 5, + 5, + 4, + 4, + 0, + 3, + 2, + 6, + 2, + 5, + 1, + 7, + 3, + 6, + 6, + 5, + 5, + 7, + 0, + 3, + 0, + 3, + 6, + 5, + 5, + 1, + 7, + 7, + 3, + 6, + 2, + 2, + 5, + 1, + 4, + 2, + 3, + 5, + 2, + 5, + 4, + 3, + 4, + 0, + 6, + 6, + 5, + 5, + 7, + 0, + 3, + 7, + 3, + 2, + 2, + 5, + 4, + 4, + 0, + 3, + 2 + ], + "resolved": { + "derived_keys": { + "key_a": null, + "key_b": null + }, + "detector": { + "alpha": null, + "context_width": 4, + "deduplicate_contexts": true, + "key_b_prf_key": null, + "p_value_threshold": 0.01, + "prf_key": "42", + "type": "GumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": "single_turn", + "deduplicate_contexts_max_history": 8, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 0, + "key_b_effective": null, + "key_b_routed": null, + "partial_context_skips": 0, + "routing_draws": 0 + } + }, + "gumbel-philox-key42-cw4-mid-no-dedup": { + "configuration": { + "detection_key": "42", + "fixture": { + "dominant_bias": 16, + "dominant_token": 3, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [], + "vocabulary_size": 8 + }, + "key": "42", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "gumbel", + "scheme_config": { + "context_width": 4, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": false, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "none", + "generation_deduplicate_contexts_max_history": null, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 64, + "p_value": 2.136704912577492e-10, + "p_value_ratio": 2.136704912577492e-08, + "score": 127.21876922102206 + }, + "generation": [ + 7, + 2, + 2, + 5, + 5, + 4, + 4, + 0, + 3, + 2, + 6, + 2, + 5, + 1, + 7, + 3, + 6, + 6, + 5, + 5, + 7, + 0, + 3, + 0, + 3, + 6, + 5, + 5, + 1, + 7, + 7, + 3, + 6, + 2, + 2, + 5, + 1, + 4, + 2, + 3, + 5, + 2, + 5, + 4, + 3, + 4, + 0, + 6, + 6, + 5, + 5, + 7, + 0, + 3, + 7, + 3, + 2, + 2, + 5, + 4, + 4, + 0, + 3, + 2 + ], + "resolved": { + "derived_keys": { + "key_a": null, + "key_b": null + }, + "detector": { + "alpha": null, + "context_width": 4, + "deduplicate_contexts": false, + "key_b_prf_key": null, + "p_value_threshold": 0.01, + "prf_key": "42", + "type": "GumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": "none", + "deduplicate_contexts_max_history": null, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 0, + "key_b_effective": null, + "key_b_routed": null, + "partial_context_skips": 0, + "routing_draws": 0 + } + }, + "gumbel-philox-key42-cw4-near-threshold": { + "configuration": { + "detection_key": "42", + "fixture": { + "dominant_bias": 0, + "dominant_token": null, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 8, + "prompt": [], + "vocabulary_size": 16 + }, + "key": "42", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "gumbel", + "scheme_config": { + "context_width": 4, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "single_turn", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.0015 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 8, + "p_value": 0.0009697880157637389, + "p_value_ratio": 0.6465253438424926, + "score": 19.672172609864035 + }, + "generation": [ + 7, + 2, + 13, + 2, + 5, + 1, + 11, + 14 + ], + "resolved": { + "derived_keys": { + "key_a": null, + "key_b": null + }, + "detector": { + "alpha": null, + "context_width": 4, + "deduplicate_contexts": true, + "key_b_prf_key": null, + "p_value_threshold": 0.0015, + "prf_key": "42", + "type": "GumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": "single_turn", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 0, + "key_b_effective": null, + "key_b_routed": null, + "partial_context_skips": 0, + "routing_draws": 0 + } + }, + "gumbel-philox-key42-cw4-wrong-key": { + "configuration": { + "detection_key": "43", + "fixture": { + "dominant_bias": 0, + "dominant_token": null, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [], + "vocabulary_size": 128 + }, + "key": "42", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "gumbel", + "scheme_config": { + "context_width": 4, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "single_turn", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": false, + "num_scored_tokens": 64, + "p_value": 0.5208447936891313, + "p_value_ratio": 52.08447936891313, + "score": 63.25096716585621 + }, + "generation": [ + 46, + 24, + 45, + 80, + 14, + 26, + 22, + 124, + 21, + 45, + 16, + 30, + 5, + 89, + 0, + 10, + 27, + 70, + 80, + 62, + 1, + 11, + 117, + 53, + 10, + 45, + 94, + 37, + 43, + 22, + 78, + 49, + 119, + 9, + 22, + 118, + 82, + 75, + 32, + 24, + 84, + 27, + 122, + 66, + 78, + 64, + 124, + 17, + 91, + 12, + 51, + 15, + 75, + 64, + 127, + 45, + 112, + 15, + 83, + 114, + 121, + 109, + 24, + 77 + ], + "resolved": { + "derived_keys": { + "key_a": null, + "key_b": null + }, + "detector": { + "alpha": null, + "context_width": 4, + "deduplicate_contexts": true, + "key_b_prf_key": null, + "p_value_threshold": 0.01, + "prf_key": "43", + "type": "GumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": "single_turn", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 0, + "key_b_effective": null, + "key_b_routed": null, + "partial_context_skips": 0, + "routing_draws": 0 + } + }, + "gumbel-philox-keymax-cw16-all": { + "configuration": { + "detection_key": "18446744073709551615", + "fixture": { + "dominant_bias": 0, + "dominant_token": null, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 64, + "prompt": [], + "vocabulary_size": 128 + }, + "key": "18446744073709551615", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "gumbel", + "scheme_config": { + "context_width": 16, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "all", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 64, + "p_value": 6.166345567165398e-36, + "p_value_ratio": 6.1663455671653984e-34, + "score": 220.2767573421712 + }, + "generation": [ + 46, + 42, + 38, + 34, + 30, + 26, + 22, + 18, + 14, + 10, + 6, + 2, + 44, + 125, + 121, + 117, + 88, + 6, + 97, + 12, + 86, + 121, + 0, + 106, + 98, + 119, + 101, + 36, + 47, + 110, + 25, + 24, + 102, + 6, + 101, + 37, + 114, + 50, + 53, + 127, + 109, + 80, + 69, + 5, + 96, + 60, + 14, + 42, + 38, + 58, + 118, + 79, + 4, + 110, + 42, + 102, + 66, + 80, + 72, + 33, + 78, + 106, + 67, + 52 + ], + "resolved": { + "derived_keys": { + "key_a": null, + "key_b": null + }, + "detector": { + "alpha": null, + "context_width": 16, + "deduplicate_contexts": true, + "key_b_prf_key": null, + "p_value_threshold": 0.01, + "prf_key": "18446744073709551615", + "type": "GumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 16, + "deduplicate_contexts": "all", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 16, + "key_b_effective": null, + "key_b_routed": null, + "partial_context_skips": 16, + "routing_draws": 0 + } + }, + "gumbel-philox-repeated-cw1": { + "configuration": { + "detection_key": "42", + "fixture": { + "dominant_bias": 52, + "dominant_token": 3, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 48, + "prompt": [], + "vocabulary_size": 16 + }, + "key": "42", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "gumbel", + "scheme_config": { + "context_width": 1, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "single_turn", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 15, + "p_value": 6.97501804959627e-05, + "p_value_ratio": 0.00697501804959627, + "score": 34.40948699058768 + }, + "generation": [ + 14, + 3, + 13, + 2, + 1, + 8, + 3, + 3, + 3, + 10, + 2, + 2, + 5, + 15, + 11, + 3, + 3, + 6, + 2, + 5, + 12, + 11, + 3, + 3, + 3, + 6, + 2, + 5, + 1, + 3, + 3, + 3, + 10, + 6, + 2, + 5, + 1, + 3, + 3, + 3, + 6, + 13, + 9, + 4, + 11, + 3, + 3, + 3 + ], + "resolved": { + "derived_keys": { + "key_a": null, + "key_b": null + }, + "detector": { + "alpha": null, + "context_width": 1, + "deduplicate_contexts": true, + "key_b_prf_key": null, + "p_value_threshold": 0.01, + "prf_key": "42", + "type": "GumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 1, + "deduplicate_contexts": "single_turn", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 33, + "key_b_effective": null, + "key_b_routed": null, + "partial_context_skips": 0, + "routing_draws": 0 + } + }, + "gumbel-philox-repeated-cw4-all": { + "configuration": { + "detection_key": "42", + "fixture": { + "dominant_bias": 52, + "dominant_token": 3, + "logit_denominator": 16, + "logit_modulus": 131, + "num_tokens": 48, + "prompt": [], + "vocabulary_size": 16 + }, + "key": "42", + "prf": "philox", + "prf_version": "philox4x32-10-v1", + "scheme": "gumbel", + "scheme_config": { + "context_width": 4, + "detection_alpha": 0.1, + "detection_deduplicate_contexts": true, + "generation_alpha": 0.1, + "generation_deduplicate_contexts": "all", + "generation_deduplicate_contexts_max_history": 8192, + "p_value_threshold": 0.01 + } + }, + "detection": { + "is_watermarked": true, + "num_scored_tokens": 47, + "p_value": 6.687298332875021e-08, + "p_value_ratio": 6.687298332875021e-06, + "score": 92.47607252164309 + }, + "generation": [ + 3, + 3, + 6, + 2, + 4, + 1, + 3, + 3, + 3, + 10, + 2, + 2, + 3, + 8, + 4, + 3, + 3, + 6, + 12, + 5, + 8, + 4, + 11, + 10, + 3, + 6, + 9, + 5, + 8, + 11, + 3, + 3, + 10, + 6, + 1, + 12, + 3, + 3, + 3, + 3, + 6, + 9, + 5, + 5, + 8, + 0, + 0, + 3 + ], + "resolved": { + "derived_keys": { + "key_a": null, + "key_b": null + }, + "detector": { + "alpha": null, + "context_width": 4, + "deduplicate_contexts": true, + "key_b_prf_key": null, + "p_value_threshold": 0.01, + "prf_key": "42", + "type": "GumbelWatermarkDetector" + }, + "watermark_config": { + "algorithm": "gumbel", + "allow_target_only_watermarking": false, + "alpha": 0.1, + "context_width": 4, + "deduplicate_contexts": "all", + "deduplicate_contexts_max_history": 8192, + "prf": "philox" + } + }, + "trace": { + "dedup_skips": 5, + "key_b_effective": null, + "key_b_routed": null, + "partial_context_skips": 4, + "routing_draws": 0 + } + } + }, + "schema_version": 3 +}