Signed-off-by: Shern Shiou Tan <shernshiou@gmail.com> Signed-off-by: Simon Veitner <sveitner@redhat.com> Signed-off-by: Raphael Rialland <raphael.rialland@mistral.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Simon Veitner <sveitner@redhat.com> Co-authored-by: Raphael Rialland <raphael.rialland@mistral.ai> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
131 lines
3.8 KiB
Python
131 lines
3.8 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
from abc import ABC, abstractmethod
|
|
from dataclasses import dataclass
|
|
from typing import Protocol, runtime_checkable
|
|
|
|
import torch
|
|
|
|
from vllm.config.watermarking import WatermarkContextScope
|
|
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class WatermarkSample:
|
|
token_ids: torch.Tensor
|
|
logits: torch.Tensor
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class RandomSampler:
|
|
expanded_idx_mapping: torch.Tensor
|
|
temperatures: torch.Tensor
|
|
seeds: torch.Tensor
|
|
positions: torch.Tensor
|
|
use_fp64: bool = False
|
|
is_drafting: bool = False
|
|
logits_cache: torch.Tensor | None = None
|
|
logits_cache_col: torch.Tensor | None = None
|
|
logits_cache_source: torch.Tensor | None = None
|
|
|
|
def __call__(self, logits: torch.Tensor) -> torch.Tensor:
|
|
return gumbel_sample(
|
|
logits,
|
|
self.expanded_idx_mapping,
|
|
self.temperatures,
|
|
self.seeds,
|
|
self.positions,
|
|
apply_temperature=False,
|
|
is_drafting=self.is_drafting,
|
|
logits_cache=self.logits_cache,
|
|
logits_cache_col=self.logits_cache_col,
|
|
logits_cache_source=self.logits_cache_source,
|
|
use_fp64=self.use_fp64,
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class DraftContextState:
|
|
contexts: torch.Tensor
|
|
enabled: torch.Tensor
|
|
prior_contexts: torch.Tensor
|
|
prompt_lens: torch.Tensor
|
|
deduplicate_contexts: WatermarkContextScope
|
|
deduplicate_contexts_max_history: int | None
|
|
all_token_ids: torch.Tensor | None = None
|
|
total_lens: torch.Tensor | None = None
|
|
|
|
|
|
class Watermarker(ABC):
|
|
@property
|
|
@abstractmethod
|
|
def context_width(self) -> int:
|
|
raise NotImplementedError
|
|
|
|
def sample(
|
|
self,
|
|
logits: torch.Tensor,
|
|
contexts: torch.Tensor,
|
|
random_sampler: RandomSampler | None = None,
|
|
skip_mask: torch.Tensor | None = None,
|
|
) -> WatermarkSample:
|
|
if skip_mask is not None:
|
|
if random_sampler is None:
|
|
raise ValueError("skip_mask requires a random sampler")
|
|
mixed = self._try_sample_mixed(logits, contexts, skip_mask, random_sampler)
|
|
if mixed is not None:
|
|
return mixed
|
|
|
|
watermarked = self._sample_watermarked(logits, contexts)
|
|
if skip_mask is None:
|
|
return watermarked
|
|
|
|
assert random_sampler is not None
|
|
token_ids = torch.where(
|
|
skip_mask, random_sampler(logits), watermarked.token_ids
|
|
)
|
|
output_logits = watermarked.logits
|
|
if output_logits is not logits:
|
|
output_logits = torch.where(skip_mask.unsqueeze(-1), logits, output_logits)
|
|
return WatermarkSample(token_ids, output_logits)
|
|
|
|
@abstractmethod
|
|
def _sample_watermarked(
|
|
self,
|
|
logits: torch.Tensor,
|
|
contexts: torch.Tensor,
|
|
) -> WatermarkSample:
|
|
raise NotImplementedError
|
|
|
|
def _try_sample_mixed(
|
|
self,
|
|
logits: torch.Tensor,
|
|
contexts: torch.Tensor,
|
|
skip_mask: torch.Tensor,
|
|
random_sampler: RandomSampler,
|
|
) -> WatermarkSample | None:
|
|
return None
|
|
|
|
def _try_sample_block(
|
|
self,
|
|
logits: torch.Tensor,
|
|
num_steps: int,
|
|
random_sampler: RandomSampler,
|
|
state: DraftContextState,
|
|
) -> torch.Tensor | None:
|
|
"""Sample a (request, step) draft block in one fused pass.
|
|
|
|
Returns None when there is no fused implementation, in which case the
|
|
caller samples the block one step at a time.
|
|
"""
|
|
return None
|
|
|
|
|
|
@runtime_checkable
|
|
class SupportsSpeculativeDecoding(Protocol):
|
|
"""A watermarker compatible with standard speculative rejection sampling."""
|
|
|
|
draft_watermarker: Watermarker
|
|
target_watermarker: Watermarker
|