[watermarking] Dual-key gumbel-max watermarking for speculative decoding support (#56122)

Signed-off-by: Raphael Rialland <raphael.rialland@mistral.ai>
Signed-off-by: Simon Veitner <sveitner@redhat.com>
Signed-off-by: Tomas Ruiz <tomas.ruiz.te@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Simon Veitner <sveitner@redhat.com>
Co-authored-by: Tomas Ruiz <tomas.ruiz.te@gmail.com>
This commit is contained in:
Raphaël Rialland
2026-09-12 18:42:01 +00:00
committed by GitHub
co-authored by OpenAI Codex Simon Veitner Tomas Ruiz
parent 1ee4be4dbc
commit 7ee8a6dd01
28 changed files with 3460 additions and 245 deletions
+62 -9
View File
@@ -50,7 +50,7 @@ from .reasoning import ReasoningConfig
from .scheduler import SchedulerConfig
from .speculative import EagleModelTypes, NgramGPUTypes, SpeculativeConfig
from .structured_outputs import StructuredOutputsConfig
from .utils import SupportsHash, config, replace
from .utils import SupportsHash, config, get_field, replace
from .watermarking import WatermarkConfig
from .weight_transfer import WeightTransferConfig
@@ -1142,14 +1142,67 @@ class VllmConfig:
watermark_config = getattr(self, "watermark_config", None)
if watermark_config is None:
return
if (
self.speculative_config is not None
and not watermark_config.supports_speculative_decoding
):
raise ValueError(
f"The {watermark_config.algorithm} watermarking algorithm "
"does not support speculative decoding."
)
if self.speculative_config is not None:
speculative_config = self.speculative_config
if speculative_config.draft_sample_method != "probabilistic":
raise ValueError(
"Speculative decoding with watermarking requires "
"draft_sample_method='probabilistic'."
)
if speculative_config.rejection_sample_method != "standard":
raise ValueError(
"Speculative decoding with watermarking requires "
"rejection_sample_method='standard'."
)
if (
speculative_config.parallel_drafting
and speculative_config.method != "dspark"
):
raise ValueError(
"Parallel speculative drafting is not supported with watermarking."
)
if speculative_config.method not in ("dspark", "eagle", "eagle3", "mtp"):
raise ValueError(
"Watermarking supports only autoregressive model-based "
"speculative decoding."
)
if (
not watermark_config.allow_target_only_watermarking
and not watermark_config.supports_speculative_decoding
):
raise ValueError(
f"The '{watermark_config.algorithm}' watermarking algorithm "
"does not support speculative decoding. Set "
"allow_target_only_watermarking=true to leave draft tokens "
"unwatermarked."
)
if (
watermark_config.allow_target_only_watermarking
and not watermark_config.supports_speculative_decoding
):
logger.warning_once(
"Target-only watermarking leaves accepted draft tokens "
"unwatermarked, weakening detectability in proportion to the "
"share of output tokens supplied by accepted drafts.",
scope="global",
)
if watermark_config.deduplicate_contexts != "none":
logger.warning_once(
"Context deduplication is not supported with speculative "
"decoding and will not be applied to accepted drafts, "
"rejection-recovery tokens, or bonus tokens.",
scope="global",
)
if watermark_config.algorithm == "dual_key_gumbel" and (
watermark_config.alpha != get_field(WatermarkConfig, "alpha").default
):
logger.warning_once(
"Speculative decoding selects the watermark key by token role: "
"draft tokens use key A, recovery and bonus tokens use key B. "
"The configured alpha=%s is not used.",
watermark_config.alpha,
scope="global",
)
if beam_search:
raise ValueError("Beam search is not supported with watermarking.")
if custom_sampler:
+49 -18
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import hashlib
from typing import Literal
from pydantic import Field, model_validator
@@ -11,12 +12,16 @@ from vllm.logger import init_logger
logger = init_logger(__name__)
WatermarkingAlgorithm = Literal["gumbel"]
WatermarkingAlgorithm = Literal["gumbel", "dual_key_gumbel"]
WatermarkPRFName = Literal["philox"]
WatermarkContextScope = Literal["none", "single_turn", "all"]
_SPECULATIVE_DECODING_SUPPORT: dict[WatermarkingAlgorithm, bool] = {
"gumbel": False,
}
_MIN_RECOMMENDED_DEDUP_HISTORY = 1024
def derive_watermark_key(key: int, domain: bytes) -> int:
digest = hashlib.sha256(domain + key.to_bytes(8, "big")).digest()
return int.from_bytes(digest[:8], "big")
@config
@@ -27,23 +32,49 @@ class WatermarkConfig:
"""Secret key used to watermark generated text."""
algorithm: WatermarkingAlgorithm = "gumbel"
"""Algorithm used to watermark generated text."""
alpha: float = Field(default=0.1, ge=0, le=1)
"""Probability of selecting key B for dual-key watermarking."""
context_width: int = Field(default=4, ge=1)
"""Number of prior output tokens used by the watermark PRF."""
"""Number of prior tokens used by the watermark PRF."""
deduplicate_contexts: WatermarkContextScope = "single_turn"
"""Which history is searched for a repeated context before a token is
sampled; a repeated context is sampled without the watermark. `none`
disables the search. `single_turn` (default) searches this request's
generated tokens. `all` also searches the prompt and samples the first
`context_width` generated tokens without watermarking."""
deduplicate_contexts_max_history: int | None = Field(default=8192, ge=1)
"""Number of most recent history positions searched (default 8192), or
`None` for the whole scope. Each position is compared over the
`context_width` tokens before it. Ignored when `deduplicate_contexts` is
`none`."""
prf: WatermarkPRFName = "philox"
"""Pseudorandom function used by the watermarking algorithm."""
@model_validator(mode="after")
def validate_key(self) -> Self:
if self.key > 2**64 - 1:
raise ValueError("philox keys must fit in 64 bits")
if self.algorithm == "gumbel":
logger.warning_once(
"Single-key Gumbel-max watermarking may increase the frequency of "
"degenerate generations, including repetition loops.",
scope="global",
)
return self
allow_target_only_watermarking: bool = False
"""Allow speculative decoding without watermarking draft tokens."""
@property
def supports_speculative_decoding(self) -> bool:
return _SPECULATIVE_DECODING_SUPPORT[self.algorithm]
return self.algorithm == "dual_key_gumbel"
@model_validator(mode="after")
def validate_watermark_settings(self) -> Self:
if self.key > 2**64 - 1:
raise ValueError("philox keys must fit in 64 bits")
history_is_too_short = (
self.deduplicate_contexts_max_history is not None
and self.deduplicate_contexts_max_history < _MIN_RECOMMENDED_DEDUP_HISTORY
)
if self.algorithm in ("gumbel", "dual_key_gumbel") and (
self.deduplicate_contexts == "none" or history_is_too_short
):
logger.warning_once(
"Gumbel-max watermarking with context deduplication "
"disabled or limited to fewer than "
f"{_MIN_RECOMMENDED_DEDUP_HISTORY} positions may increase the "
"frequency of degenerate generations, including repetition loops. "
"Use deduplicate_contexts='single_turn' or 'all' with "
"deduplicate_contexts_max_history at least "
f"{_MIN_RECOMMENDED_DEDUP_HISTORY} or null to mitigate this.",
scope="global",
)
return self
@@ -44,6 +44,9 @@ from vllm.model_executor.warmup.replayssm_warmup import (
from vllm.model_executor.warmup.spec_decode_rejection_warmup import (
spec_decode_rejection_warmup,
)
from vllm.model_executor.warmup.watermark_sample_warmup import (
watermark_sample_warmup,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import is_deep_gemm_supported
from vllm.utils.flashinfer import has_flashinfer
@@ -207,6 +210,7 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
if enable_jit_warmup:
kimi_k3_triton_warmup(worker)
spec_decode_rejection_warmup(worker)
watermark_sample_warmup(worker)
qwen4_exp_qsa_triton_warmup(worker)
if enable_jit_warmup and current_platform.is_device_capability_family(100):
@@ -1,17 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Warm up spec-decode rejection-sampler Triton kernels.
The rejection sampler kernels (``_compute_local_logits_stats_kernel``,
``_rejection_kernel``, ``_resample_kernel``) are JIT-compiled by Triton on
first use. Without warmup, the first spec-decode request pays a multi-second
compilation cost. This pre-compiles them with dummy data matching the
server's vocab size and speculative config.
"""
"""Warm up speculative-decoding rejection kernels."""
from __future__ import annotations
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
import torch
@@ -39,42 +32,76 @@ def spec_decode_rejection_warmup(worker: Worker) -> None:
if num_spec <= 0 or vocab_size <= 0:
return
# Mirror the constexpr-relevant flags the runtime uses.
rejection_method = getattr(spec_config, "rejection_sample_method", None)
use_block_verification = rejection_method == "block"
use_synthetic = rejection_method == "synthetic"
# HAS_DRAFT_LOGITS is a constexpr, so the warmup must match the runtime.
use_draft_logits = (
getattr(spec_config, "draft_sample_method", "greedy") == "probabilistic"
)
device = worker.device
num_reqs = 1
tokens_per_req = num_spec + 1
num_logits = num_reqs * tokens_per_req
# Triton JIT-specializes on tensor dtypes. The target logits may be fp32
# (apply_sampling_params copies to fp32 when processing is needed) or the
# model dtype (pass-through otherwise), while draft logits are always the
# model dtype. Warm every (target, draft) combination the runtime can hit.
watermark_config = getattr(worker.vllm_config, "watermark_config", None)
watermark_kwargs: dict[str, Any] = {}
if watermark_config is not None:
from vllm.v1.watermarking.spec_decode import speculative_target_watermark_key
try:
watermark_key = speculative_target_watermark_key(watermark_config)
except Exception:
logger.warning(
"Skipping spec-decode rejection sampler warmup: could not derive "
"the watermark recovery key.",
exc_info=True,
)
return
# Tensor dtypes and integer widths affect Triton specialization.
watermark_kwargs = {
"contexts": torch.zeros(
(num_logits, watermark_config.context_width),
dtype=torch.int32,
device=device,
),
"watermarking": torch.zeros(num_reqs, dtype=torch.bool, device=device),
"watermark_key": watermark_key,
}
# Sampling-parameter processing may promote either distribution to fp32.
model_dtype = model_config.dtype
warmup_dtype_pairs = {
(model_dtype, model_dtype),
(torch.float32, torch.float32),
(torch.float32, model_dtype),
(model_dtype, torch.float32),
}
warmup_dtype_pairs: set[tuple[torch.dtype, torch.dtype | None]]
if use_draft_logits:
warmup_dtype_pairs = {
(model_dtype, model_dtype),
(torch.float32, torch.float32),
(torch.float32, model_dtype),
(model_dtype, torch.float32),
}
else:
warmup_dtype_pairs = {(model_dtype, None), (torch.float32, None)}
logger.info(
"Warming up spec-decode rejection sampler kernels "
"(vocab=%d, num_spec=%d, dtype_pairs=%s, block_verify=%s).",
"(vocab=%d, num_spec=%d, dtype_pairs=%s, block_verify=%s, watermark=%s).",
vocab_size,
num_spec,
[(str(t), str(d)) for t, d in warmup_dtype_pairs],
use_block_verification,
watermark_config is not None,
)
for tgt_dtype, draft_dtype in warmup_dtype_pairs:
target_logits = torch.zeros(
(num_logits, vocab_size), dtype=tgt_dtype, device=device
)
draft_logits = torch.zeros(
(num_reqs, num_spec, vocab_size), dtype=draft_dtype, device=device
draft_logits = (
torch.zeros(
(num_reqs, num_spec, vocab_size), dtype=draft_dtype, device=device
)
if draft_dtype is not None
else None
)
synthetic_rates = (
torch.full((num_spec,), 0.5, dtype=torch.float32, device=device)
@@ -85,14 +112,14 @@ def spec_decode_rejection_warmup(worker: Worker) -> None:
rejection_sample(
target_logits=target_logits,
draft_logits=draft_logits,
draft_sampled=torch.zeros(num_logits, dtype=torch.int64, device=device),
draft_sampled=torch.zeros(num_logits, dtype=torch.int32, device=device),
cu_num_logits=torch.tensor(
[0, num_logits], dtype=torch.int32, device=device
),
pos=torch.zeros(num_logits, dtype=torch.int64, device=device),
idx_mapping=torch.zeros(num_reqs, dtype=torch.int32, device=device),
idx_mapping=torch.zeros(num_reqs, dtype=torch.int64, device=device),
expanded_idx_mapping=torch.zeros(
num_logits, dtype=torch.int32, device=device
num_logits, dtype=torch.int64, device=device
),
expanded_local_pos=torch.arange(
num_logits, dtype=torch.int32, device=device
@@ -103,6 +130,7 @@ def spec_decode_rejection_warmup(worker: Worker) -> None:
synthetic_conditional_rates=synthetic_rates,
use_fp64=False,
use_block_verification=use_block_verification,
**watermark_kwargs,
)
except Exception:
logger.warning(
@@ -0,0 +1,155 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Warm up the watermarked sampler's Triton kernel.
``_philox_gumbel_kernel`` is JIT-compiled once per (key, logits dtype,
skip-mask, ``USE_FP64``) specialization. The generic sampler warmup samples with
``SamplingParams.for_sampler_warmup()``, whose feature-heavy logits processing
forces the fp32 copy in ``apply_sampling_params``, so the first ordinary
temperature-1.0 request is the first launch with model-dtype logits and pays
the compilation inside inference. This pre-compiles every specialization the
sampler path can launch for the configured watermark.
"""
from __future__ import annotations
from typing import TYPE_CHECKING, cast
import torch
from vllm.logger import init_logger
from vllm.platforms import current_platform
if TYPE_CHECKING:
from vllm.v1.watermarking.watermarker import Watermarker
from vllm.v1.worker.gpu_worker import Worker
logger = init_logger(__name__)
_NUM_WARMUP_TOKENS = 1
def _philox_key(watermarker: Watermarker) -> int | None:
"""None when the watermarker takes the torch path and compiles no kernel."""
from vllm.v1.watermarking.gumbel import GumbelWatermarker
from vllm.v1.watermarking.prfs import PhiloxPRF
if not isinstance(watermarker, GumbelWatermarker):
return None
prf = watermarker.prf
return prf.key if type(prf) is PhiloxPRF else None
def _philox_sampler_keys(watermarker: Watermarker) -> list[int]:
"""Philox keys the sampler launches the kernel with."""
from vllm.v1.watermarking.gumbel import DualKeyGumbelWatermarker
watermarkers: list[Watermarker] = [watermarker]
if isinstance(watermarker, DualKeyGumbelWatermarker):
# ``DualKeyGumbelWatermarker.sample`` launches once per key, and
# collapses to a single key at the alpha bounds, where the other key's
# specialization is never launched.
if watermarker.alpha == 0:
watermarkers = [watermarker.draft_watermarker]
elif watermarker.alpha == 1:
watermarkers = [watermarker.target_watermarker]
else:
watermarkers = [
watermarker.draft_watermarker,
watermarker.target_watermarker,
]
return [key for key in map(_philox_key, watermarkers) if key is not None]
@torch.inference_mode()
def watermark_sample_warmup(worker: Worker) -> None:
if getattr(worker.vllm_config, "watermark_config", None) is None:
return
# ``GumbelWatermarker`` launches the kernel only on CUDA
# (vllm/v1/watermarking/gumbel.py:63,:82); elsewhere the sampler takes the
# torch path and warming here would compile a kernel nothing launches.
if not current_platform.is_cuda_alike():
return
from vllm.v1.watermarking.gpu_sampler import GPUWatermarkSampler
from vllm.v1.worker.gpu.sample.watermark import philox_gumbel_sample
# Read the sampler the runner built instead of rebuilding it from config:
# its watermarker (the target role under a speculative config), its context
# deduplication setting and its fp64 flag are what the launches have to
# match, and cannot drift from the runner. A rank that does not sample -- a
# non-final pipeline stage, a pooling model -- holds no such sampler.
sampler = getattr(worker.model_runner, "sampler", None)
if not isinstance(sampler, GPUWatermarkSampler):
return
model_config = worker.vllm_config.model_config
device = worker.device
try:
keys = _philox_sampler_keys(sampler.watermarker)
if not keys:
return
# The sampler passes model-dtype logits through when no logits
# processing is needed and fp32 otherwise. ``ModelConfig.dtype`` is
# resolved to a torch dtype during validation.
dtypes = {cast("torch.dtype", model_config.dtype), torch.float32}
# ``GPUWatermarkSampler._sample_random`` builds a skip mask whenever
# context deduplication is on; with it off only a batch that mixes in an
# opted-out or greedy row does, and the mask-free specialization stays
# reachable.
with_skip_mask = (
(True,) if sampler.deduplicate_contexts != "none" else (True, False)
)
vocab_size = model_config.get_vocab_size()
contexts = torch.zeros(
(_NUM_WARMUP_TOKENS, sampler.watermarker.context_width),
dtype=torch.int32,
device=device,
)
# Dtypes mirror the runtime buffers the sampler passes and are part of
# the specialization key: int32 request-state token ids, int64 index and
# seed buffers, fp32 sampling temperatures.
sampling_state = {
"skip_mask": torch.zeros(
_NUM_WARMUP_TOKENS, dtype=torch.bool, device=device
),
"expanded_idx_mapping": torch.zeros(
_NUM_WARMUP_TOKENS, dtype=torch.int64, device=device
),
"temperatures": torch.ones(
_NUM_WARMUP_TOKENS, dtype=torch.float32, device=device
),
"seeds": torch.zeros(_NUM_WARMUP_TOKENS, dtype=torch.int64, device=device),
"positions": torch.zeros(
_NUM_WARMUP_TOKENS, dtype=torch.int64, device=device
),
}
logger.info(
"Warming up watermark sampler kernel (vocab=%d, keys=%d, dtypes=%s, "
"skip_mask=%s).",
vocab_size,
len(keys),
[str(dtype) for dtype in dtypes],
list(with_skip_mask),
)
for dtype in dtypes:
logits = torch.zeros(
(_NUM_WARMUP_TOKENS, vocab_size), dtype=dtype, device=device
)
for key in keys:
for use_skip_mask in with_skip_mask:
philox_gumbel_sample(
logits,
contexts,
key,
# Only the mixed path forwards the sampler's setting;
# ``GumbelWatermarker._sample_watermarked`` launches
# the mask-free variant with the fp32 default.
use_fp64=use_skip_mask and sampler.use_fp64_gumbel,
**(sampling_state if use_skip_mask else {}),
)
except Exception:
logger.warning("Skipping watermark sampler warmup.", exc_info=True)
+12 -1
View File
@@ -1,9 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.config.watermarking import derive_watermark_key
from vllm.v1.watermarking.detector import WatermarkDetection, WatermarkDetector
from vllm.v1.watermarking.factory import create_watermarker
from vllm.v1.watermarking.gumbel import (
DualKeyGumbelWatermarkDetector,
DualKeyGumbelWatermarker,
GumbelWatermarkDetector,
GumbelWatermarker,
)
@@ -12,12 +15,19 @@ from vllm.v1.watermarking.prfs import (
WatermarkPRF,
create_prf,
)
from vllm.v1.watermarking.watermarker import Watermarker, WatermarkSample
from vllm.v1.watermarking.watermarker import (
SupportsSpeculativeDecoding,
Watermarker,
WatermarkSample,
)
__all__ = [
"DualKeyGumbelWatermarkDetector",
"DualKeyGumbelWatermarker",
"GumbelWatermarkDetector",
"GumbelWatermarker",
"PhiloxPRF",
"SupportsSpeculativeDecoding",
"WatermarkDetection",
"WatermarkDetector",
"WatermarkPRF",
@@ -25,4 +35,5 @@ __all__ = [
"WatermarkSample",
"create_watermarker",
"create_prf",
"derive_watermark_key",
]
+14 -16
View File
@@ -1,24 +1,22 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from vllm.config.watermarking import WatermarkConfig
from vllm.v1.watermarking.gumbel import GumbelWatermarker
from vllm.v1.watermarking.gumbel import (
DualKeyGumbelWatermarker,
GumbelWatermarker,
)
from vllm.v1.watermarking.watermarker import Watermarker
def _create_gumbel(config: WatermarkConfig) -> Watermarker:
return GumbelWatermarker(config.key, config.context_width, config.prf)
_WATERMARKERS: dict[str, Callable[[WatermarkConfig], Watermarker]] = {
"gumbel": _create_gumbel,
}
def create_watermarker(config: WatermarkConfig) -> Watermarker:
watermarker_factory = _WATERMARKERS.get(config.algorithm)
if watermarker_factory is None:
raise ValueError(f"Unknown watermarking algorithm: {config.algorithm}")
return watermarker_factory(config)
if config.algorithm == "gumbel":
return GumbelWatermarker(config.key, config.context_width, config.prf)
if config.algorithm == "dual_key_gumbel":
return DualKeyGumbelWatermarker(
config.key,
config.context_width,
config.prf,
config.alpha,
)
raise ValueError(f"Unknown watermarking algorithm: {config.algorithm}")
+98 -35
View File
@@ -1,16 +1,18 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Literal
import numpy as np
import torch
from vllm.logger import init_logger
from vllm.sampling_params import SamplingParams
from vllm.v1.sample.ops.topk_topp_sampler import apply_top_k_top_p
from vllm.v1.watermarking.watermarker import Watermarker
from vllm.v1.watermarking.watermarker import RandomSampler, Watermarker
from vllm.v1.worker.gpu.buffer_utils import UvaBackedTensor
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
from vllm.v1.worker.gpu.sample.sampler import Sampler
from vllm.v1.worker.gpu.sample.watermark import repeated_context_mask
logger = init_logger(__name__)
@@ -20,10 +22,14 @@ class GPUWatermarkSampler(Sampler):
self,
watermarker: Watermarker,
*args,
deduplicate_contexts: Literal["none", "single_turn", "all"] = "single_turn",
deduplicate_contexts_max_history: int | None = 8192,
**kwargs,
) -> None:
super().__init__(*args, **kwargs)
self.watermarker = watermarker
self.deduplicate_contexts = deduplicate_contexts
self.deduplicate_contexts_max_history = deduplicate_contexts_max_history
self.watermarking = UvaBackedTensor(
self.sampling_states.max_num_reqs, dtype=torch.bool
)
@@ -72,37 +78,38 @@ class GPUWatermarkSampler(Sampler):
processed_logits = apply_top_k_top_p(processed_logits, top_k, top_p)
contexts = self._get_contexts(expanded_idx_mapping)
def random_sample(sample_logits: torch.Tensor) -> torch.Tensor:
return gumbel_sample(
sample_logits,
expanded_idx_mapping,
self.sampling_states.temperature.gpu,
self.sampling_states.seeds.gpu,
pos,
apply_temperature=False,
is_drafting=False,
use_fp64=self.use_fp64_gumbel,
repeated_contexts = None
if self.deduplicate_contexts != "none":
repeated_contexts = self._get_repeated_contexts(
expanded_idx_mapping, contexts
)
temperatures = self.sampling_states.temperature.gpu[expanded_idx_mapping]
needs_mixed_sampling = repeated_contexts is not None or not np.all(enabled)
skip_mask = None
if needs_mixed_sampling:
watermarking = self.watermarking.gpu[expanded_idx_mapping] & (
temperatures != 0
)
if repeated_contexts is not None:
watermarking &= ~repeated_contexts
skip_mask = ~watermarking
random_sampler = RandomSampler(
expanded_idx_mapping=expanded_idx_mapping,
temperatures=self.sampling_states.temperature.gpu,
seeds=self.sampling_states.seeds.gpu,
positions=pos,
use_fp64=self.use_fp64_gumbel,
)
output = self.watermarker.sample(
processed_logits,
contexts,
random_sample,
random_sampler,
skip_mask=skip_mask,
)
temperatures = self.sampling_states.temperature.gpu[expanded_idx_mapping]
watermarking = self.watermarking.gpu[expanded_idx_mapping] & (temperatures != 0)
if not np.all(enabled):
unwatermarked = random_sample(processed_logits)
sampled = torch.where(watermarking, output.token_ids, unwatermarked)
output_logits = output.logits
if output.logits is not processed_logits:
output_logits = torch.where(
watermarking.unsqueeze(-1), output.logits, processed_logits
)
else:
sampled = output.token_ids
output_logits = output.logits
sampled = output.token_ids
output_logits = output.logits
sampled = torch.where(
temperatures == 0,
processed_logits.argmax(dim=-1),
@@ -110,22 +117,78 @@ class GPUWatermarkSampler(Sampler):
)
return sampled, output_logits
def _get_contexts(self, expanded_idx_mapping: torch.Tensor) -> torch.Tensor:
def _get_repeated_contexts(
self,
expanded_idx_mapping: torch.Tensor,
contexts: torch.Tensor,
) -> torch.Tensor:
return repeated_context_mask(
self.req_states.all_token_ids.gpu,
expanded_idx_mapping,
self.req_states.prompt_len.gpu,
self.req_states.total_len.gpu,
contexts,
self.deduplicate_contexts_max_history,
include_prompt=self.deduplicate_contexts == "all",
skip_partial_context=self.deduplicate_contexts == "all",
)
def _get_contexts(
self,
expanded_idx_mapping: torch.Tensor,
expanded_local_pos: torch.Tensor | None = None,
draft_sampled: torch.Tensor | None = None,
) -> torch.Tensor:
"""Build contexts for flattened verification rows.
With committed tokens ``[10, 11]``, drafts ``[20, 21]``, and width 2,
local positions ``[0, 1, 2]`` use contexts ``[10, 11]``, ``[11, 20]``,
and ``[20, 21]``. The first verification row precedes the first draft.
"""
context_width = self.watermarker.context_width
req_indices = expanded_idx_mapping.to(torch.int64)
valid_reqs = req_indices >= 0
safe_req_indices = req_indices.clamp_min(0)
total_lens = self.req_states.total_len.gpu[safe_req_indices].to(torch.int64)
prompt_lens = self.req_states.prompt_len.gpu[safe_req_indices].to(torch.int64)
if expanded_local_pos is None:
expanded_local_pos = torch.zeros_like(req_indices)
else:
expanded_local_pos = expanded_local_pos.to(torch.int64)
offsets = torch.arange(
-context_width, 0, dtype=torch.int64, device=req_indices.device
)
positions = total_lens.unsqueeze(-1) + offsets
valid_positions = valid_reqs.unsqueeze(-1) & (
positions >= prompt_lens.unsqueeze(-1)
positions = (
total_lens.unsqueeze(-1) + expanded_local_pos.unsqueeze(-1) + offsets
)
positions = positions.clamp_min(0)
contexts = self.req_states.all_token_ids.gpu[
safe_req_indices.unsqueeze(-1), positions
committed = positions < total_lens.unsqueeze(-1)
valid_committed = (
valid_reqs.unsqueeze(-1)
& committed
& (positions >= prompt_lens.unsqueeze(-1))
)
committed_contexts = self.req_states.all_token_ids.gpu[
safe_req_indices.unsqueeze(-1),
positions.clamp(0, self.req_states.all_token_ids.gpu.shape[1] - 1),
]
return torch.where(valid_positions, contexts, -1)
contexts = torch.where(valid_committed, committed_contexts, -1)
if draft_sampled is None:
return contexts
row_indices = torch.arange(
len(req_indices), dtype=torch.int64, device=req_indices.device
)
draft_offsets = positions - total_lens.unsqueeze(-1)
draft_indices = (
row_indices.unsqueeze(-1)
- expanded_local_pos.unsqueeze(-1)
+ draft_offsets
+ 1
)
valid_drafts = (
valid_reqs.unsqueeze(-1)
& ~committed
& (draft_offsets < expanded_local_pos.unsqueeze(-1))
)
draft_contexts = draft_sampled[draft_indices.clamp(0, len(draft_sampled) - 1)]
return torch.where(valid_drafts, draft_contexts, contexts)
+162 -3
View File
@@ -8,13 +8,14 @@ import warnings
import torch
from vllm.config.watermarking import WatermarkPRFName
from vllm.config.watermarking import WatermarkPRFName, derive_watermark_key
from vllm.v1.watermarking.detector import (
WatermarkDetector,
)
from vllm.v1.watermarking.prfs import PhiloxPRF, WatermarkPRF, create_prf
from vllm.v1.watermarking.watermarker import (
RandomSampler,
SupportsSpeculativeDecoding,
Watermarker,
WatermarkSample,
)
@@ -54,11 +55,10 @@ class GumbelWatermarker(Watermarker):
def context_width(self) -> int:
return self._context_width
def sample(
def _sample_watermarked(
self,
logits: torch.Tensor,
contexts: torch.Tensor,
_random_sample: RandomSampler,
) -> WatermarkSample:
if type(self.prf) is PhiloxPRF and logits.device.type == "cuda":
from vllm.v1.worker.gpu.sample.watermark import philox_gumbel_sample
@@ -72,6 +72,110 @@ class GumbelWatermarker(Watermarker):
noise = -torch.log(-torch.log(uniforms))
return WatermarkSample(torch.argmax(logits + noise, dim=-1), logits)
def _try_sample_mixed(
self,
logits: torch.Tensor,
contexts: torch.Tensor,
skip_mask: torch.Tensor,
random_sampler: RandomSampler,
) -> WatermarkSample | None:
if type(self.prf) is not PhiloxPRF or logits.device.type != "cuda":
return None
from vllm.v1.worker.gpu.sample.watermark import philox_gumbel_sample
token_ids = philox_gumbel_sample(
logits,
contexts,
self.prf.key,
skip_mask=skip_mask,
expanded_idx_mapping=random_sampler.expanded_idx_mapping,
temperatures=random_sampler.temperatures,
seeds=random_sampler.seeds,
positions=random_sampler.positions,
use_fp64=random_sampler.use_fp64,
)
return WatermarkSample(token_ids, logits)
class DualKeyGumbelWatermarker(Watermarker, SupportsSpeculativeDecoding):
def __init__(
self,
key: int,
context_width: int = 4,
prf: WatermarkPRFName = "philox",
alpha: float = 0.1,
) -> None:
if not 0 <= alpha <= 1:
raise ValueError("alpha must be between 0 and 1")
self.alpha = alpha
self.draft_watermarker = GumbelWatermarker(
derive_watermark_key(key, b"key_a"), context_width, prf
)
self.target_watermarker = GumbelWatermarker(
derive_watermark_key(key, b"key_b"), context_width, prf
)
self._routing_logits_cache: dict[torch.device, torch.Tensor] = {}
def _routing_logits(self, device: torch.device) -> torch.Tensor:
# Building this tensor on every CUDA call stages a pageable H2D copy and
# synchronizes the stream.
cached = self._routing_logits_cache.get(device)
if cached is None:
cached = torch.tensor(
[1 - self.alpha, self.alpha],
dtype=torch.float32,
device=device,
).log()
self._routing_logits_cache[device] = cached
return cached
@property
def context_width(self) -> int:
return self.draft_watermarker.context_width
def sample(
self,
logits: torch.Tensor,
contexts: torch.Tensor,
random_sampler: RandomSampler | None = None,
skip_mask: torch.Tensor | None = None,
) -> WatermarkSample:
if self.alpha == 0:
return self.draft_watermarker.sample(
logits, contexts, random_sampler, skip_mask
)
if self.alpha == 1:
return self.target_watermarker.sample(
logits, contexts, random_sampler, skip_mask
)
if random_sampler is None:
raise ValueError("dual-key Gumbel routing requires a random sampler")
key_a_sample = self.draft_watermarker.sample(
logits, contexts, random_sampler, skip_mask
)
key_b_sample = self.target_watermarker.sample(
logits, contexts, random_sampler, skip_mask
)
routing_logits = self._routing_logits(logits.device).expand(logits.shape[0], -1)
use_key_a = random_sampler(routing_logits) == 0
return WatermarkSample(
torch.where(
use_key_a,
key_a_sample.token_ids,
key_b_sample.token_ids,
),
logits,
)
def _sample_watermarked(
self,
logits: torch.Tensor,
contexts: torch.Tensor,
) -> WatermarkSample:
raise ValueError("dual-key Gumbel routing requires a random sampler")
class GumbelWatermarkDetector(WatermarkDetector):
def __init__(
@@ -100,6 +204,61 @@ class GumbelWatermarkDetector(WatermarkDetector):
return token_scores.sum().item()
class DualKeyGumbelWatermarkDetector(GumbelWatermarkDetector):
"""Detect a dual-key watermark with weighted early fusion.
``alpha`` is the weight assigned to key B; key A receives ``1 - alpha``.
"""
def __init__(
self,
key: int,
context_width: int = 4,
p_value_threshold: float = 0.01,
prf: WatermarkPRFName = "philox",
deduplicate_contexts: bool = True,
alpha: float = 0.2,
) -> None:
if not 0 <= alpha <= 1:
raise ValueError("alpha must be between 0 and 1")
super().__init__(
derive_watermark_key(key, b"key_a"),
context_width,
p_value_threshold,
prf,
deduplicate_contexts,
)
self.key_b_prf = create_prf(prf, derive_watermark_key(key, b"key_b"))
self.alpha = alpha
def _score_tokens(
self, contexts: torch.Tensor, targets: torch.Tensor
) -> torch.Tensor:
scores = torch.stack(
[
-torch.log1p(
-prf.uniform(contexts, targets.unsqueeze(-1))
.squeeze(-1)
.to(torch.float64)
)
for prf in (self.prf, self.key_b_prf)
],
dim=-1,
)
weights = scores.new_tensor([1 - self.alpha, self.alpha])
return scores @ weights
def _get_p_value(self, score: float, num_scored_tokens: int) -> float:
variance = (1 - self.alpha) ** 2 + self.alpha**2
if self.alpha in (0, 0.5, 1):
return _gamma_survival_integer_shape(
score / variance, round(num_scored_tokens / variance)
)
shape = torch.tensor(num_scored_tokens / variance, dtype=torch.float64)
scaled_score = torch.tensor(score / variance, dtype=torch.float64)
return torch.special.gammaincc(shape, scaled_score).item()
def _validate_context_width(context_width: int) -> None:
if context_width < 1:
raise ValueError("context_width must be positive")
+153
View File
@@ -0,0 +1,153 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.config.watermarking import WatermarkConfig
from vllm.v1.watermarking.factory import create_watermarker
from vllm.v1.watermarking.gumbel import GumbelWatermarker
from vllm.v1.watermarking.prfs import PhiloxPRF
from vllm.v1.watermarking.watermarker import (
SupportsSpeculativeDecoding,
Watermarker,
)
from vllm.v1.worker.gpu.spec_decode.rejection_sampler_utils import rejection_sample
class DraftWatermarker:
def __init__(
self,
watermarker: Watermarker,
max_num_reqs: int,
device: torch.device,
) -> None:
self.watermarker = watermarker
self.contexts = torch.zeros(
max_num_reqs,
watermarker.context_width,
dtype=torch.int64,
device=device,
)
self.enabled = torch.zeros(max_num_reqs, dtype=torch.bool, device=device)
def prepare(self, contexts: torch.Tensor, enabled: torch.Tensor) -> None:
num_reqs = contexts.shape[0]
self.contexts[:num_reqs].copy_(contexts)
self.enabled[:num_reqs].copy_(enabled)
def sample(
self,
logits: torch.Tensor,
ordinary_sampled: torch.Tensor,
idx_mapping: torch.Tensor,
temperature: torch.Tensor,
) -> torch.Tensor:
request_temperatures = temperature[idx_mapping]
processed_logits = logits / torch.where(
request_temperatures == 0, 1, request_temperatures
).unsqueeze(-1)
watermarked = self.watermarker.sample(
processed_logits,
self.contexts[: logits.shape[0]],
).token_ids
enabled = self.enabled[: logits.shape[0]] & (request_temperatures != 0)
sampled = torch.where(enabled, watermarked, ordinary_sampled)
contexts = self.contexts[: logits.shape[0]]
contexts.copy_(torch.cat((contexts[:, 1:], sampled.unsqueeze(-1)), dim=-1))
return sampled
def create_speculative_target_watermarker(watermarker: Watermarker) -> Watermarker:
if isinstance(watermarker, SupportsSpeculativeDecoding):
return watermarker.target_watermarker
return watermarker
def create_speculative_draft_watermarker(
watermarker: Watermarker,
max_num_reqs: int,
device: torch.device,
allow_target_only: bool,
) -> DraftWatermarker | None:
if isinstance(watermarker, SupportsSpeculativeDecoding):
return DraftWatermarker(watermarker.draft_watermarker, max_num_reqs, device)
if allow_target_only:
return None
raise ValueError(
f"The {type(watermarker).__name__} watermarking algorithm does not support "
"speculative decoding. Set allow_target_only_watermarking=true to "
"leave draft tokens unwatermarked."
)
def _philox_key(watermarker: Watermarker) -> int:
if not (
isinstance(watermarker, GumbelWatermarker)
and type(watermarker.prf) is PhiloxPRF
):
raise NotImplementedError(
"In-kernel watermarked recovery supports only the Philox PRF "
"GumbelWatermarker"
)
return watermarker.prf.key
def _resolve_watermark_key(watermarker: Watermarker) -> int:
"""Return the Philox recovery key."""
if isinstance(watermarker, SupportsSpeculativeDecoding):
raise ValueError(
f"{type(watermarker).__name__} keys the target role separately from "
"the draft. Pass create_speculative_target_watermarker(watermarker) "
"so the recovery draw carries the target's key."
)
return _philox_key(watermarker)
def speculative_target_watermark_key(
watermark_config: WatermarkConfig | None,
) -> int | None:
"""Resolve the resample kernel's Philox key from configuration."""
if watermark_config is None:
return None
return _resolve_watermark_key(
create_speculative_target_watermarker(create_watermarker(watermark_config))
)
def watermarked_rejection_sample(
target_logits: torch.Tensor,
draft_logits: torch.Tensor | None,
draft_sampled: torch.Tensor,
cu_num_logits: torch.Tensor,
pos: torch.Tensor,
idx_mapping: torch.Tensor,
expanded_idx_mapping: torch.Tensor,
expanded_local_pos: torch.Tensor,
temperature: torch.Tensor,
seed: torch.Tensor,
num_speculative_steps: int,
contexts: torch.Tensor,
watermarking: torch.Tensor,
watermarker: Watermarker,
use_fp64: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Use the target key for rejection recovery and bonus tokens."""
assert contexts.shape == (target_logits.shape[0], watermarker.context_width)
return rejection_sample(
target_logits,
draft_logits,
draft_sampled,
cu_num_logits,
pos,
idx_mapping,
expanded_idx_mapping,
expanded_local_pos,
temperature,
seed,
num_speculative_steps,
use_fp64=use_fp64,
contexts=contexts,
watermarking=watermarking,
watermark_key=_resolve_watermark_key(watermarker),
)
+68 -5
View File
@@ -2,13 +2,12 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from collections.abc import Callable
from dataclasses import dataclass
from typing import TypeAlias
from typing import Protocol, runtime_checkable
import torch
RandomSampler: TypeAlias = Callable[[torch.Tensor], torch.Tensor]
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
@dataclass(frozen=True)
@@ -17,17 +16,81 @@ class WatermarkSample:
logits: torch.Tensor
@dataclass(frozen=True)
class RandomSampler:
expanded_idx_mapping: torch.Tensor
temperatures: torch.Tensor
seeds: torch.Tensor
positions: torch.Tensor
use_fp64: bool = False
def __call__(self, logits: torch.Tensor) -> torch.Tensor:
return gumbel_sample(
logits,
self.expanded_idx_mapping,
self.temperatures,
self.seeds,
self.positions,
apply_temperature=False,
is_drafting=False,
use_fp64=self.use_fp64,
)
class Watermarker(ABC):
@property
@abstractmethod
def context_width(self) -> int:
raise NotImplementedError
@abstractmethod
def sample(
self,
logits: torch.Tensor,
contexts: torch.Tensor,
random_sample: RandomSampler,
random_sampler: RandomSampler | None = None,
skip_mask: torch.Tensor | None = None,
) -> WatermarkSample:
if skip_mask is not None:
if random_sampler is None:
raise ValueError("skip_mask requires a random sampler")
mixed = self._try_sample_mixed(logits, contexts, skip_mask, random_sampler)
if mixed is not None:
return mixed
watermarked = self._sample_watermarked(logits, contexts)
if skip_mask is None:
return watermarked
assert random_sampler is not None
token_ids = torch.where(
skip_mask, random_sampler(logits), watermarked.token_ids
)
output_logits = watermarked.logits
if output_logits is not logits:
output_logits = torch.where(skip_mask.unsqueeze(-1), logits, output_logits)
return WatermarkSample(token_ids, output_logits)
@abstractmethod
def _sample_watermarked(
self,
logits: torch.Tensor,
contexts: torch.Tensor,
) -> WatermarkSample:
raise NotImplementedError
def _try_sample_mixed(
self,
logits: torch.Tensor,
contexts: torch.Tensor,
skip_mask: torch.Tensor,
random_sampler: RandomSampler,
) -> WatermarkSample | None:
return None
@runtime_checkable
class SupportsSpeculativeDecoding(Protocol):
"""A watermarker compatible with standard speculative rejection sampling."""
draft_watermarker: Watermarker
target_watermarker: Watermarker
+29 -2
View File
@@ -79,6 +79,10 @@ from vllm.v1.outputs import (
)
from vllm.v1.watermarking import create_watermarker
from vllm.v1.watermarking.gpu_sampler import GPUWatermarkSampler
from vllm.v1.watermarking.spec_decode import (
create_speculative_target_watermarker,
speculative_target_watermark_key,
)
from vllm.v1.worker.block_table import get_block_table_width
from vllm.v1.worker.cp_utils import check_attention_cp_compatibility
from vllm.v1.worker.gpu import pcp_manager as pcp
@@ -472,8 +476,18 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if self.vllm_config.watermark_config is None:
self.sampler = Sampler(**sampler_kwargs)
else:
watermarker = create_watermarker(self.vllm_config.watermark_config)
self.sampler = GPUWatermarkSampler(watermarker, **sampler_kwargs)
wm_config = self.vllm_config.watermark_config
watermarker = create_watermarker(wm_config)
if self.speculative_config is not None:
watermarker = create_speculative_target_watermarker(watermarker)
self.sampler = GPUWatermarkSampler(
watermarker,
deduplicate_contexts=wm_config.deduplicate_contexts,
deduplicate_contexts_max_history=(
wm_config.deduplicate_contexts_max_history
),
**sampler_kwargs,
)
custom = self.model_state.custom_sampler(self.sampler)
if custom:
@@ -484,6 +498,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.sampler,
self.speculative_config,
self.device,
watermark_key=speculative_target_watermark_key(
self.vllm_config.watermark_config
),
)
self.prompt_logprobs_worker = PromptLogprobsWorker(
self.max_num_reqs,
@@ -860,6 +877,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if hasattr(self.model, "get_mtp_target_hidden_states"):
pre_hc_hidden_states = self.model.get_mtp_target_hidden_states()
spec_hidden_states = pre_hc_hidden_states[: hidden_states.shape[0]] # type: ignore[union-attr]
if isinstance(self.sampler, GPUWatermarkSampler):
self.speculator.prepare_watermarking(
self.sampler._get_contexts(input_batch.idx_mapping),
self.sampler.watermarking.gpu[input_batch.idx_mapping],
)
with use_workspace_lane(self._draft_workspace_lane):
self.speculator.propose(
input_batch=input_batch,
@@ -2086,6 +2108,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if hasattr(self.model, "get_mtp_target_hidden_states"):
pre_hc_hidden_states = self.model.get_mtp_target_hidden_states()
spec_hidden_states = pre_hc_hidden_states[: draft_hidden_states.size(0)]
if isinstance(self.sampler, GPUWatermarkSampler):
self.speculator.prepare_watermarking(
self.sampler._get_contexts(input_batch.idx_mapping),
self.sampler.watermarking.gpu[input_batch.idx_mapping],
)
with use_workspace_lane(self._draft_workspace_lane):
draft_tokens = self.speculator.propose(
input_batch,
+326 -47
View File
@@ -25,6 +25,7 @@ from vllm.v1.watermarking.prfs.philox import (
from vllm.v1.watermarking.prfs.philox import (
_UINT32_MASK as _UINT32_MASK_VALUE,
)
from vllm.v1.worker.gpu.sample.gumbel import gumbel_noised_argmax
if HAS_TRITON:
from triton.language import math as tl_math
@@ -42,6 +43,162 @@ _CONTEXT_DOMAIN = (
_TOKEN_DOMAIN = tl.constexpr(_TOKEN_DOMAIN_VALUE) if HAS_TRITON else _TOKEN_DOMAIN_VALUE
@triton.jit
def _repeated_context_mask_kernel(
output_ptr,
all_token_ids_ptr,
all_token_ids_stride,
req_indices_ptr,
prompt_lens_ptr,
total_lens_ptr,
contexts_ptr,
context_stride,
CONTEXT_WIDTH: tl.constexpr,
MAX_HISTORY: tl.constexpr,
INCLUDE_PROMPT: tl.constexpr,
SKIP_PARTIAL_CONTEXT: tl.constexpr,
BLOCK: tl.constexpr,
):
row = tl.program_id(0).to(tl.int64)
req_idx = tl.load(req_indices_ptr + row).to(tl.int64)
if req_idx < 0:
tl.store(output_ptr + row, 0)
return
prompt_len = tl.load(prompt_lens_ptr + req_idx)
total_len = tl.load(total_lens_ptr + req_idx)
if SKIP_PARTIAL_CONTEXT and total_len - prompt_len < CONTEXT_WIDTH:
tl.store(output_ptr + row, 1)
return
sequence_start = 0 if INCLUDE_PROMPT else prompt_len
history_len = total_len - sequence_start
offsets = tl.arange(0, BLOCK)
repeated = tl.full((), 0, tl.int32)
scan_start = 0
if MAX_HISTORY > 0:
scan_start = tl.maximum(history_len - MAX_HISTORY, 0)
# Aligned loop bounds let the per-lane offsets fold into immediates.
aligned_start = (scan_start // BLOCK) * BLOCK
for block_start in tl.range(aligned_start, history_len, BLOCK):
previous_pos = block_start + offsets
in_window = (previous_pos >= scan_start) & (previous_pos < history_len)
matches = in_window
for offset in range(CONTEXT_WIDTH):
context_token = tl.load(contexts_ptr + row * context_stride + offset)
historical_pos = previous_pos + offset - CONTEXT_WIDTH
historical_token = tl.load(
all_token_ids_ptr
+ req_idx * all_token_ids_stride
+ sequence_start
+ tl.maximum(historical_pos, 0),
mask=in_window & (historical_pos >= 0),
other=-1,
)
matches &= historical_token == context_token
repeated |= tl.max(matches.to(tl.int32), axis=0)
tl.store(output_ptr + row, repeated)
def _repeated_context_mask_cpu(
all_token_ids: torch.Tensor,
req_indices: torch.Tensor,
prompt_lens: torch.Tensor,
total_lens: torch.Tensor,
contexts: torch.Tensor,
max_history: int | None = None,
include_prompt: bool = False,
skip_partial_context: bool = False,
) -> torch.Tensor:
"""Reference implementation; the parity tests check the Triton kernel against it."""
repeated = torch.zeros(len(req_indices), dtype=torch.bool)
for row, req_idx_tensor in enumerate(req_indices):
req_idx = int(req_idx_tensor)
if req_idx < 0:
continue
prompt_len = int(prompt_lens[req_idx])
total_len = int(total_lens[req_idx])
if skip_partial_context and total_len - prompt_len < contexts.shape[-1]:
repeated[row] = True
continue
sequence_start = 0 if include_prompt else prompt_len
history_tokens = all_token_ids[req_idx, sequence_start:total_len].tolist()
prefix = [-1] * contexts.shape[-1]
current_context = tuple(contexts[row].tolist())
history_start = (
max(0, len(history_tokens) - max_history) if max_history is not None else 0
)
for history_pos, token_id in enumerate(history_tokens):
if history_pos >= history_start and (
tuple(prefix[-contexts.shape[-1] :]) == current_context
):
repeated[row] = True
break
prefix.append(token_id)
return repeated
def repeated_context_mask(
all_token_ids: torch.Tensor,
req_indices: torch.Tensor,
prompt_lens: torch.Tensor,
total_lens: torch.Tensor,
contexts: torch.Tensor,
max_history: int | None = None,
include_prompt: bool = False,
skip_partial_context: bool = False,
) -> torch.Tensor:
"""Return, per row, whether the row's context already occurred in its history.
Args:
all_token_ids: `[max_num_reqs, max_model_len]` token ids of every request.
req_indices: Request slot per sampled row; -1 marks a padding row, which
is reported as not repeated.
prompt_lens: Prompt length per request slot.
total_lens: Prompt plus generated length per request slot.
contexts: `[num_rows, context_width]` context per row, padded with -1
before the start of the scanned history.
max_history: Number of most recent history positions searched, or
``None`` for all of them. The window compared at each position
reaches `context_width` tokens further back.
include_prompt: Search the prompt as well as the generated tokens.
skip_partial_context: Mark contexts containing start padding so they use
ordinary sampling.
"""
if max_history is not None and max_history < 1:
raise ValueError("max_history must be positive or None")
if all_token_ids.device.type == "cpu":
return _repeated_context_mask_cpu(
all_token_ids,
req_indices,
prompt_lens,
total_lens,
contexts,
max_history,
include_prompt,
skip_partial_context,
)
if contexts.stride(-1) != 1:
contexts = contexts.contiguous()
repeated = torch.empty(len(req_indices), dtype=torch.bool, device=contexts.device)
_repeated_context_mask_kernel[(len(req_indices),)](
repeated,
all_token_ids,
all_token_ids.stride(0),
req_indices,
prompt_lens,
total_lens,
contexts,
contexts.stride(0),
CONTEXT_WIDTH=contexts.shape[-1],
MAX_HISTORY=0 if max_history is None else max_history,
INCLUDE_PROMPT=include_prompt,
SKIP_PARTIAL_CONTEXT=skip_partial_context,
BLOCK=512,
)
return repeated
@triton.jit
def _mulhilo32(multiplier: tl.constexpr, value):
high = tl_math.umulhi(multiplier, value)
@@ -73,12 +230,100 @@ def _uint32_to_uniform(value):
return (scaled.to(tl.uint32, bitcast=True) - 1).to(tl.float32, bitcast=True)
@triton.jit
def _philox_context_state(
contexts_row_ptr,
key_0,
key_1,
CONTEXT_WIDTH: tl.constexpr,
):
state_0 = tl.full((), _CONTEXT_DOMAIN, tl.uint32)
state_1 = tl.full((), CONTEXT_WIDTH, tl.uint32)
state_2 = tl.full((), 0, tl.uint32)
state_3 = tl.full((), 0, tl.uint32)
for offset in range(0, CONTEXT_WIDTH, 4):
context_0 = tl.load(contexts_row_ptr + offset).to(tl.uint32)
if offset + 1 < CONTEXT_WIDTH:
context_1 = tl.load(contexts_row_ptr + offset + 1).to(tl.uint32)
else:
context_1 = tl.full((), _UINT32_MASK - 1, tl.uint32)
if offset + 2 < CONTEXT_WIDTH:
context_2 = tl.load(contexts_row_ptr + offset + 2).to(tl.uint32)
else:
context_2 = tl.full((), _UINT32_MASK - 2, tl.uint32)
if offset + 3 < CONTEXT_WIDTH:
context_3 = tl.load(contexts_row_ptr + offset + 3).to(tl.uint32)
else:
context_3 = tl.full((), _UINT32_MASK - 3, tl.uint32)
state_0, state_1, state_2, state_3 = _philox4x32_10(
(state_0 ^ context_0) & _UINT32_MASK,
(state_1 ^ context_1) & _UINT32_MASK,
(state_2 ^ context_2) & _UINT32_MASK,
(state_3 ^ context_3) & _UINT32_MASK,
(key_0 ^ offset) & _UINT32_MASK,
key_1,
)
return state_0, state_1, state_2, state_3
@triton.jit
def _philox_candidate_words(groups, state_0, state_1, state_2, state_3, key_0, key_1):
candidate_words = groups.to(tl.uint32)
vector_zero = candidate_words * 0
return _philox4x32_10(
candidate_words & _UINT32_MASK,
state_0 + vector_zero,
state_1 + vector_zero,
state_2 + vector_zero,
((key_0 ^ state_3) & _UINT32_MASK) + vector_zero,
((key_1 ^ _TOKEN_DOMAIN) & _UINT32_MASK) + vector_zero,
)
@triton.jit
def _philox_gumbel_from_logits(logits, word):
logits = tl.where(logits != logits, float("-inf"), logits)
uniform = _uint32_to_uniform(word)
return logits - tl.log(-tl.log(uniform))
@triton.jit
def _gumbel_value(logits_ptr, output, mask):
logits = tl.load(logits_ptr, mask=mask, other=float("-inf")).to(tl.float32)
logits = tl.where(logits != logits, float("-inf"), logits)
uniform = _uint32_to_uniform(output)
return logits - tl.log(-tl.log(uniform))
return _philox_gumbel_from_logits(logits, output)
@triton.jit
def philox_gumbel_block_argmax(
logits,
mask,
block_idx,
contexts_row_ptr,
key_0,
key_1,
CONTEXT_WIDTH: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
"""Apply the detector-compatible PRF to one vocabulary block."""
tl.static_assert(BLOCK_SIZE % 4 == 0)
state_0, state_1, state_2, state_3 = _philox_context_state(
contexts_row_ptr, key_0, key_1, CONTEXT_WIDTH
)
groups = block_idx * (BLOCK_SIZE // 4) + tl.arange(0, BLOCK_SIZE // 4)
output_0, output_1, output_2, output_3 = _philox_candidate_words(
groups, state_0, state_1, state_2, state_3, key_0, key_1
)
# Lay the per-group words out in token order: 4g+0, 4g+1, 4g+2, 4g+3.
words = tl.interleave(
tl.interleave(output_0, output_2),
tl.interleave(output_1, output_3),
)
values = tl.where(
mask,
_philox_gumbel_from_logits(logits.to(tl.float32), words),
float("-inf"),
)
return tl.max(values, axis=0, return_indices=True)
@triton.jit
@@ -100,60 +345,63 @@ def _philox_gumbel_kernel(
logits_stride,
contexts_ptr,
context_stride,
skip_mask_ptr,
expanded_idx_mapping_ptr,
seeds_ptr,
pos_ptr,
temp_ptr,
key_0_value,
key_1_value,
vocab_size,
CONTEXT_WIDTH: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
USE_FP64: tl.constexpr,
):
row = tl.program_id(0).to(tl.int64)
block_index = tl.program_id(1)
groups = block_index * (BLOCK_SIZE // 4) + tl.arange(0, BLOCK_SIZE // 4)
if skip_mask_ptr is not None:
req_state_idx = tl.load(expanded_idx_mapping_ptr + row).to(tl.int64)
valid_req = req_state_idx >= 0
if tl.load(skip_mask_ptr + row):
temp = tl.load(temp_ptr + req_state_idx, mask=valid_req, other=0.0).to(
tl.float32
)
seed = tl.load(seeds_ptr + req_state_idx, mask=valid_req, other=0)
pos = tl.load(pos_ptr + row)
candidate = block_index * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = candidate < vocab_size
logits_row = logits_ptr + row * logits_stride
logits = tl.load(logits_row + candidate, mask=mask, other=float("-inf")).to(
tl.float32
)
value, index = gumbel_noised_argmax(
logits,
candidate,
mask,
seed,
pos,
temp,
IS_DRAFTING=False,
USE_FP64=USE_FP64,
APPLY_TEMPERATURE=False,
)
token_id = block_index * BLOCK_SIZE + index
tl.store(
local_argmax_ptr + row * local_argmax_stride + block_index,
token_id,
)
tl.store(local_max_ptr + row * local_max_stride + block_index, value)
return
key_0 = key_0_value.to(tl.uint32)
key_1 = key_1_value.to(tl.uint32)
state_0 = tl.full((), _CONTEXT_DOMAIN, tl.uint32)
state_1 = tl.full((), CONTEXT_WIDTH, tl.uint32)
state_2 = tl.full((), 0, tl.uint32)
state_3 = tl.full((), 0, tl.uint32)
for offset in range(0, CONTEXT_WIDTH, 4):
context_0 = tl.load(contexts_ptr + row * context_stride + offset).to(tl.uint32)
if offset + 1 < CONTEXT_WIDTH:
context_1 = tl.load(contexts_ptr + row * context_stride + offset + 1).to(
tl.uint32
)
else:
context_1 = tl.full((), _UINT32_MASK - 1, tl.uint32)
if offset + 2 < CONTEXT_WIDTH:
context_2 = tl.load(contexts_ptr + row * context_stride + offset + 2).to(
tl.uint32
)
else:
context_2 = tl.full((), _UINT32_MASK - 2, tl.uint32)
if offset + 3 < CONTEXT_WIDTH:
context_3 = tl.load(contexts_ptr + row * context_stride + offset + 3).to(
tl.uint32
)
else:
context_3 = tl.full((), _UINT32_MASK - 3, tl.uint32)
state_0, state_1, state_2, state_3 = _philox4x32_10(
(state_0 ^ context_0) & _UINT32_MASK,
(state_1 ^ context_1) & _UINT32_MASK,
(state_2 ^ context_2) & _UINT32_MASK,
(state_3 ^ context_3) & _UINT32_MASK,
(key_0 ^ offset) & _UINT32_MASK,
key_1,
)
candidate_words = groups.to(tl.uint32)
vector_zero = candidate_words * 0
output_0, output_1, output_2, output_3 = _philox4x32_10(
candidate_words & _UINT32_MASK,
state_0 + vector_zero,
state_1 + vector_zero,
state_2 + vector_zero,
((key_0 ^ state_3) & _UINT32_MASK) + vector_zero,
((key_1 ^ _TOKEN_DOMAIN) & _UINT32_MASK) + vector_zero,
state_0, state_1, state_2, state_3 = _philox_context_state(
contexts_ptr + row * context_stride, key_0, key_1, CONTEXT_WIDTH
)
output_0, output_1, output_2, output_3 = _philox_candidate_words(
groups, state_0, state_1, state_2, state_3, key_0, key_1
)
candidate_0 = groups * 4
candidate_1 = candidate_0 + 1
@@ -184,17 +432,42 @@ def _philox_gumbel_kernel(
def philox_gumbel_sample(
logits: torch.Tensor, contexts: torch.Tensor, key: int
logits: torch.Tensor,
contexts: torch.Tensor,
key: int,
*,
skip_mask: torch.Tensor | None = None,
expanded_idx_mapping: torch.Tensor | None = None,
temperatures: torch.Tensor | None = None,
seeds: torch.Tensor | None = None,
positions: torch.Tensor | None = None,
use_fp64: bool = False,
) -> torch.Tensor:
sampling_state = (expanded_idx_mapping, temperatures, seeds, positions)
if skip_mask is None and any(value is not None for value in sampling_state):
raise ValueError("sampling state requires skip_mask")
if skip_mask is not None and any(value is None for value in sampling_state):
raise ValueError("skip_mask requires complete sampling state")
if logits.stride(-1) != 1:
logits = logits.contiguous()
if contexts.stride(-1) != 1:
contexts = contexts.contiguous()
if skip_mask is not None:
skip_mask = skip_mask.contiguous()
if expanded_idx_mapping is not None:
expanded_idx_mapping = expanded_idx_mapping.contiguous()
if positions is not None:
positions = positions.contiguous()
num_tokens, vocab_size = logits.shape
block_size = 1024
num_blocks = triton.cdiv(vocab_size, block_size)
local_argmax = logits.new_empty(num_tokens, num_blocks, dtype=torch.int64)
local_max = logits.new_empty(num_tokens, num_blocks, dtype=torch.float32)
local_max = logits.new_empty(
num_tokens,
num_blocks,
dtype=torch.float64 if use_fp64 else torch.float32,
)
_philox_gumbel_kernel[(num_tokens, num_blocks)](
local_argmax,
local_argmax.stride(0),
@@ -204,11 +477,17 @@ def philox_gumbel_sample(
logits.stride(0),
contexts,
contexts.stride(0),
skip_mask,
expanded_idx_mapping,
seeds,
positions,
temperatures,
key & _UINT32_MASK_VALUE,
key >> 32,
vocab_size,
CONTEXT_WIDTH=contexts.shape[-1],
BLOCK_SIZE=block_size,
USE_FP64=use_fp64,
)
max_block_index = local_max.argmax(dim=-1, keepdim=True)
return local_argmax.gather(dim=-1, index=max_block_index).view(-1)
@@ -137,7 +137,7 @@ class DSparkSpeculator(DFlashSpeculator):
# sample_pos is the predicted token's position P. Sampling keys a draw
# by the position before the sampled token, P-1.
return gumbel_sample(
sampled = gumbel_sample(
logits,
idx_map,
self.temperature,
@@ -149,6 +149,14 @@ class DSparkSpeculator(DFlashSpeculator):
logits_cache_col=self._step_cols[step],
use_fp64=self.use_fp64_gumbel,
)
if self.draft_watermarker is not None:
sampled = self.draft_watermarker.sample(
logits,
sampled,
idx_map,
self.temperature,
)
return sampled
def _sample_sequential(self, num_reqs: int, head_hidden: torch.Tensor) -> None:
if self._draft_topk is not None:
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable, Iterator
from typing import Any
import numpy as np
import torch
@@ -10,6 +11,7 @@ from vllm.config.model import PROCESSED_LOGPROBS_MODES
from vllm.triton_utils import tl, triton
from vllm.v1.outputs import LogprobsTensors
from vllm.v1.spec_decode.utils import unconditional_to_conditional_rates
from vllm.v1.watermarking.gpu_sampler import GPUWatermarkSampler
from vllm.v1.worker.gpu.input_batch import (
InputBatch,
get_num_sampled_and_rejected,
@@ -78,8 +80,13 @@ class RejectionSampler:
sampler: Sampler,
spec_config: SpeculativeConfig,
device: torch.device,
*,
watermark_key: int | None = None,
):
self.sampler = sampler
self.watermark_key = watermark_key
if watermark_key is not None:
assert isinstance(sampler, GPUWatermarkSampler)
self.num_speculative_steps = spec_config.num_speculative_tokens
self.enable_adaptive_verification = spec_config.enable_adaptive_verification
rejection_sample_method = spec_config.rejection_sample_method
@@ -141,6 +148,23 @@ class RejectionSampler:
in ("raw_logits", "processed_logits"),
)
def _watermarking_kwargs(
self,
draft_sampled: torch.Tensor,
expanded_idx_mapping: torch.Tensor,
expanded_local_pos: torch.Tensor,
) -> dict[str, Any]:
if self.watermark_key is None:
return {}
assert isinstance(self.sampler, GPUWatermarkSampler)
return {
"contexts": self.sampler._get_contexts(
expanded_idx_mapping, expanded_local_pos, draft_sampled
),
"watermarking": self.sampler.watermarking.gpu,
"watermark_key": self.watermark_key,
}
def _verify(
self,
logits: torch.Tensor,
@@ -177,6 +201,9 @@ class RejectionSampler:
self.synthetic_conditional_rates,
use_fp64=self.sampler.use_fp64_gumbel,
use_block_verification=self.use_block_verification,
**self._watermarking_kwargs(
draft_sampled, expanded_idx_mapping, expanded_local_pos
),
)
return processed_logits, sampled, num_sampled
@@ -4,6 +4,7 @@ import torch
from vllm.triton_utils import tl, tldevice, triton
from vllm.v1.worker.gpu.sample.gumbel import gumbel_block_argmax, tl_rand32
from vllm.v1.worker.gpu.sample.watermark import philox_gumbel_block_argmax
@triton.jit
@@ -693,6 +694,39 @@ def _rejection_kernel(
@triton.jit
def _seeded_resample_argmax(
residual_logits,
block,
mask,
resample_token_idx,
expanded_idx_mapping_ptr,
temp_ptr,
seed_ptr,
pos_ptr,
vocab_size,
USE_FP64: tl.constexpr,
):
return gumbel_block_argmax(
residual_logits,
block,
mask,
resample_token_idx,
expanded_idx_mapping_ptr,
temp_ptr,
seed_ptr,
pos_ptr,
None, # logits_cache_ptr
0, # logits_cache_stride_0
0, # logits_cache_stride_1
None, # logits_cache_col_ptr
vocab_size,
IS_DRAFTING=False,
APPLY_TEMPERATURE=False,
USE_FP64=USE_FP64,
)
@triton.jit(do_not_specialize=["watermark_key_0", "watermark_key_1"])
def _resample_kernel(
# [num_reqs, num_blocks]
resampled_local_argmax_ptr,
@@ -727,11 +761,20 @@ def _resample_kernel(
pos_ptr,
# [num_logits]
cumulative_log_p_ptr,
# [num_logits, CONTEXT_WIDTH]
contexts_ptr,
contexts_stride,
# [max_num_reqs], uint8 view of a bool tensor
watermarking_ptr,
watermark_key_0,
watermark_key_1,
vocab_size,
BLOCK_SIZE: tl.constexpr,
HAS_DRAFT_LOGITS: tl.constexpr,
USE_FP64: tl.constexpr,
USE_BLOCK_VERIFICATION: tl.constexpr,
CONTEXT_WIDTH: tl.constexpr,
WATERMARK: tl.constexpr,
):
req_idx = tl.program_id(0)
resample_idx = tl.load(rejected_step_ptr + req_idx)
@@ -763,10 +806,7 @@ def _resample_kernel(
other=float("-inf"),
).to(tl.float32)
# Compute the residual logits to resample the rejected token from.
if is_bonus or not is_valid_rejected_draft:
# Bonus token (no rejections) or -1 placeholder token. In either case,
# directly use the target logits.
residual_logits = target_logits
elif HAS_DRAFT_LOGITS:
# draft_logits is stored pre-temperature, so apply scale first.
@@ -796,12 +836,7 @@ def _resample_kernel(
)
target_log_probs += log_p_tau
draft_log_probs = draft_logits - draft_lse
# Compute the residual:
# r(x) = max(p(x) - q(x), 0)
# Gumbel sampling needs logits, so we compute it in log space:
# log(r(x)) = log(max(exp(log_p(x)) - exp(log_q(x)), 0))
# The more numerically stable form is:
# log(max(exp(a) - exp(b), 0)) = a + log(max(1 - exp(b - a), 0))
# Compute log(max(p - q, 0)) without subtracting probabilities.
ratio = tl.exp(draft_log_probs - target_log_probs)
residual_logits = tl.where(
ratio < 1.0,
@@ -809,38 +844,62 @@ def _resample_kernel(
float("-inf"),
).to(tl.float32)
else:
# One-hot draft. The residual is just the target distribution with
# the rejected draft token probability zeroed out.
# NOTE: During block verification, the residual becomes:
# 0 if x == rejected_draft_token
# p_tau * M_b(x) / Z otherwise
# Therefore p_tau is a constant that cancels under normalization,
# and does not need to be applied.
# The block-verification factor is constant and cancels on normalization.
residual_logits = tl.where(
block != rejected_draft_token,
target_logits,
float("-inf"),
).to(tl.float32)
# Resample the rejected/bonus token.
value, idx = gumbel_block_argmax(
residual_logits,
block,
mask,
resample_token_idx,
expanded_idx_mapping_ptr,
temp_ptr,
seed_ptr,
pos_ptr,
None, # logits_cache_ptr
0, # logits_cache_stride_0
0, # logits_cache_stride_1
None, # logits_cache_col_ptr
vocab_size,
IS_DRAFTING=False,
APPLY_TEMPERATURE=False,
USE_FP64=USE_FP64,
)
if WATERMARK:
# Padded and greedy rows retain the stock draw.
is_watermarked = (
tl.load(
watermarking_ptr + req_state_idx,
mask=req_state_idx >= 0,
other=0,
)
!= 0
)
if is_watermarked & (temp != 0.0):
watermark_value, idx = philox_gumbel_block_argmax(
residual_logits,
mask,
block_idx,
contexts_ptr + resample_token_idx * contexts_stride,
watermark_key_0.to(tl.uint32),
watermark_key_1.to(tl.uint32),
CONTEXT_WIDTH,
BLOCK_SIZE,
)
# Detector compatibility fixes the keyed draw at fp32.
value = watermark_value.to(tl.float64) if USE_FP64 else watermark_value
else:
value, idx = _seeded_resample_argmax(
residual_logits,
block,
mask,
resample_token_idx,
expanded_idx_mapping_ptr,
temp_ptr,
seed_ptr,
pos_ptr,
vocab_size,
USE_FP64=USE_FP64,
)
else:
value, idx = _seeded_resample_argmax(
residual_logits,
block,
mask,
resample_token_idx,
expanded_idx_mapping_ptr,
temp_ptr,
seed_ptr,
pos_ptr,
vocab_size,
USE_FP64=USE_FP64,
)
token_id = block_idx * BLOCK_SIZE + idx
tl.store(
resampled_local_argmax_ptr
@@ -947,6 +1006,9 @@ def rejection_sample(
synthetic_conditional_rates: torch.Tensor | None = None,
use_fp64: bool = False,
use_block_verification: bool = False,
contexts: torch.Tensor | None = None,
watermarking: torch.Tensor | None = None,
watermark_key: int | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
assert target_logits.ndim == 2 and target_logits.stride(-1) == 1
assert draft_logits is None or (
@@ -954,6 +1016,30 @@ def rejection_sample(
)
num_reqs = cu_num_logits.shape[0] - 1
num_logits, vocab_size = target_logits.shape
watermark = contexts is not None
assert watermark == (watermarking is not None) == (watermark_key is not None), (
"contexts, watermarking and watermark_key must be set together."
)
contexts_stride = 0
context_width = 1
watermarking_bytes: torch.Tensor | None = None
watermark_key_0 = 0
watermark_key_1 = 0
if contexts is not None:
assert watermarking is not None and watermark_key is not None
assert contexts.ndim == 2 and contexts.shape[0] == num_logits
# Context words are hashed as uint32, so int32 (the request-state token
# dtype) and int64 both land on the same PRF stream, -1 included.
assert contexts.dtype in (torch.int32, torch.int64)
if contexts.stride(-1) != 1:
contexts = contexts.contiguous()
assert watermarking.ndim == 1 and watermarking.dtype == torch.bool
contexts_stride = contexts.stride(0)
context_width = contexts.shape[-1]
watermarking_bytes = watermarking.view(torch.uint8)
watermark_key_0 = watermark_key & 0xFFFFFFFF
watermark_key_1 = watermark_key >> 32
draft_logits_stride_0 = 0
draft_logits_stride_1 = 0
if has_draft_logits := draft_logits is not None:
@@ -1165,11 +1251,18 @@ def rejection_sample(
seed,
pos,
cumulative_log_p,
contexts,
contexts_stride,
watermarking_bytes,
watermark_key_0,
watermark_key_1,
vocab_size,
BLOCK_SIZE=RESAMPLE_BLOCK_SIZE,
HAS_DRAFT_LOGITS=has_draft_logits,
USE_FP64=use_fp64,
USE_BLOCK_VERIFICATION=use_block_verification,
CONTEXT_WIDTH=context_width,
WATERMARK=watermark,
)
# Insert the resampled tokens into the output sampled.
+32 -1
View File
@@ -17,6 +17,11 @@ from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.models import supports_multimodal_embeddings
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.watermarking import create_watermarker
from vllm.v1.watermarking.spec_decode import (
DraftWatermarker,
create_speculative_draft_watermarker,
)
from vllm.v1.worker.gpu.attn_utils import (
build_attn_metadata,
init_attn_backend,
@@ -168,6 +173,17 @@ class DraftModelSpeculator(BaseSpeculator):
device=device,
)
self.draft_watermarker: DraftWatermarker | None = None
watermark_config = getattr(vllm_config, "watermark_config", None)
if watermark_config is not None:
watermarker = create_watermarker(watermark_config)
self.draft_watermarker = create_speculative_draft_watermarker(
watermarker,
self.max_num_reqs,
device,
watermark_config.allow_target_only_watermarking,
)
self.supports_mm_inputs = False
self.pcp_manager: PCPManager | None = None
@@ -373,7 +389,7 @@ class DraftModelSpeculator(BaseSpeculator):
) -> torch.Tensor:
if draft_logits is not None:
logits = self.model.compute_logits(hidden_states)
return gumbel_sample(
sampled = gumbel_sample(
logits,
idx_mapping,
temperature,
@@ -385,8 +401,23 @@ class DraftModelSpeculator(BaseSpeculator):
logits_cache_col=draft_step,
use_fp64=self.use_fp64_gumbel,
)
if self.draft_watermarker is not None:
sampled = self.draft_watermarker.sample(
logits,
sampled,
idx_mapping,
temperature,
)
return sampled
return self._greedy_sample_draft(hidden_states)
def prepare_watermarking(
self, contexts: torch.Tensor, watermarking: torch.Tensor
) -> None:
if self.draft_watermarker is None:
return
self.draft_watermarker.prepare(contexts, watermarking)
def _copy_request_inputs(
self,
num_reqs: int,