[watermarking] add context deduplication support to speculative decoding (#56807)

Signed-off-by: Raphael Rialland <raphael.rialland@mistral.ai>
Signed-off-by: Simon Veitner <sveitner@redhat.com>
Co-authored-by: Simon Veitner <sveitner@redhat.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Raphaël Rialland
2026-09-30 12:22:10 -04:00
committed by GitHub
co-authored by Simon Veitner Lucas Wilkinson
parent 73a5831127
commit 0f8b398158
17 changed files with 1379 additions and 88 deletions
-7
View File
@@ -1344,13 +1344,6 @@ class VllmConfig:
"share of output tokens supplied by accepted drafts.",
scope="global",
)
if watermark_config.deduplicate_contexts != "none":
logger.warning_once(
"Context deduplication is not supported with speculative "
"decoding and will not be applied to accepted drafts, "
"rejection-recovery tokens, or bonus tokens.",
scope="global",
)
if watermark_config.algorithm == "dual_key_gumbel" and (
watermark_config.alpha != get_field(WatermarkConfig, "alpha").default
):
+11 -2
View File
@@ -119,8 +119,9 @@ class GPUWatermarkSampler(Sampler):
self,
expanded_idx_mapping: torch.Tensor,
contexts: torch.Tensor,
expanded_local_pos: torch.Tensor | None = None,
) -> torch.Tensor:
return repeated_context_mask(
repeated = repeated_context_mask(
self.req_states.all_token_ids.gpu,
expanded_idx_mapping,
self.req_states.prompt_len.gpu,
@@ -128,8 +129,16 @@ class GPUWatermarkSampler(Sampler):
contexts,
self.deduplicate_contexts_max_history,
include_prompt=self.deduplicate_contexts == "all",
skip_partial_context=self.deduplicate_contexts == "all",
skip_partial_context=(
self.deduplicate_contexts == "all" and expanded_local_pos is None
),
history_offsets=expanded_local_pos,
local_positions=expanded_local_pos,
num_speculative_steps=(
self.num_speculative_tokens if expanded_local_pos is not None else 0
),
)
return repeated
def _get_contexts(
self,
+4
View File
@@ -94,6 +94,10 @@ class GumbelWatermarker(Watermarker):
seeds=random_sampler.seeds,
positions=random_sampler.positions,
use_fp64=random_sampler.use_fp64,
is_drafting=random_sampler.is_drafting,
logits_cache=random_sampler.logits_cache,
logits_cache_col=random_sampler.logits_cache_col,
logits_cache_source=random_sampler.logits_cache_source,
)
return WatermarkSample(token_ids, logits)
+112 -10
View File
@@ -3,14 +3,16 @@
import torch
from vllm.config.watermarking import WatermarkConfig
from vllm.config.watermarking import WatermarkConfig, WatermarkContextScope
from vllm.v1.watermarking.factory import create_watermarker
from vllm.v1.watermarking.gumbel import GumbelWatermarker
from vllm.v1.watermarking.prfs import PhiloxPRF
from vllm.v1.watermarking.watermarker import (
RandomSampler,
SupportsSpeculativeDecoding,
Watermarker,
)
from vllm.v1.worker.gpu.sample.watermark import draft_watermarking_mask
from vllm.v1.worker.gpu.spec_decode.rejection_sampler_utils import rejection_sample
@@ -20,8 +22,14 @@ class DraftWatermarker:
watermarker: Watermarker,
max_num_reqs: int,
device: torch.device,
num_speculative_steps: int,
deduplicate_contexts: WatermarkContextScope,
deduplicate_contexts_max_history: int | None,
) -> None:
self.watermarker = watermarker
self.num_speculative_steps = num_speculative_steps
self.deduplicate_contexts = deduplicate_contexts
self.deduplicate_contexts_max_history = deduplicate_contexts_max_history
self.contexts = torch.zeros(
max_num_reqs,
watermarker.context_width,
@@ -29,30 +37,112 @@ class DraftWatermarker:
device=device,
)
self.enabled = torch.zeros(max_num_reqs, dtype=torch.bool, device=device)
self.prior_contexts = torch.zeros(
max_num_reqs,
num_speculative_steps,
watermarker.context_width,
dtype=torch.int64,
device=device,
)
self.all_token_ids: torch.Tensor | None = None
self.prompt_lens: torch.Tensor | None = None
self.total_lens: torch.Tensor | None = None
def prepare(self, contexts: torch.Tensor, enabled: torch.Tensor) -> None:
def prepare(
self,
contexts: torch.Tensor,
enabled: torch.Tensor,
all_token_ids: torch.Tensor,
prompt_lens: torch.Tensor,
total_lens: torch.Tensor,
) -> None:
num_reqs = contexts.shape[0]
self.contexts[:num_reqs].copy_(contexts)
self.enabled[:num_reqs].copy_(enabled)
self.all_token_ids = all_token_ids
self.prompt_lens = prompt_lens
self.total_lens = total_lens
def _sampling_state(
self,
logits: torch.Tensor,
idx_mapping: torch.Tensor,
request_temperatures: torch.Tensor,
draft_step: int | torch.Tensor,
contexts: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
num_rows = logits.shape[0]
steps = torch.as_tensor(draft_step, device=logits.device, dtype=torch.int64)
if steps.ndim == 0:
steps = steps.expand(num_rows)
enabled = self.enabled[:num_rows] & (request_temperatures != 0)
if self.deduplicate_contexts == "none":
return steps, enabled
assert self.all_token_ids is not None
assert self.prompt_lens is not None
assert self.total_lens is not None
enabled = draft_watermarking_mask(
self.all_token_ids,
idx_mapping,
self.prompt_lens,
self.total_lens,
self.prior_contexts,
contexts,
steps,
enabled,
self.deduplicate_contexts_max_history,
include_prompt=self.deduplicate_contexts == "all",
)
return steps, enabled
def sample(
self,
logits: torch.Tensor,
ordinary_sampled: torch.Tensor,
idx_mapping: torch.Tensor,
temperature: torch.Tensor,
seed: torch.Tensor,
pos: torch.Tensor,
apply_temperature: bool,
is_drafting: bool,
logits_cache: torch.Tensor | None = None,
logits_cache_col: torch.Tensor | None = None,
use_fp64: bool = False,
logits_cache_source: torch.Tensor | None = None,
) -> torch.Tensor:
assert apply_temperature
assert is_drafting
assert logits_cache is not None
assert logits_cache_col is not None
num_rows = logits.shape[0]
request_temperatures = temperature[idx_mapping]
contexts = self.contexts[:num_rows]
steps, enabled = self._sampling_state(
logits, idx_mapping, request_temperatures, logits_cache_col, contexts
)
processed_logits = logits / torch.where(
request_temperatures == 0, 1, request_temperatures
).unsqueeze(-1)
watermarked = self.watermarker.sample(
random_sampler = RandomSampler(
expanded_idx_mapping=idx_mapping,
temperatures=temperature,
seeds=seed,
positions=pos,
use_fp64=use_fp64,
is_drafting=True,
logits_cache=logits_cache,
logits_cache_col=steps,
logits_cache_source=(
logits if logits_cache_source is None else logits_cache_source
),
)
sampled = self.watermarker.sample(
processed_logits,
self.contexts[: logits.shape[0]],
contexts,
random_sampler=random_sampler,
skip_mask=~enabled,
).token_ids
enabled = self.enabled[: logits.shape[0]] & (request_temperatures != 0)
sampled = torch.where(enabled, watermarked, ordinary_sampled)
contexts = self.contexts[: logits.shape[0]]
contexts.copy_(torch.cat((contexts[:, 1:], sampled.unsqueeze(-1)), dim=-1))
return sampled
@@ -68,9 +158,19 @@ def create_speculative_draft_watermarker(
max_num_reqs: int,
device: torch.device,
allow_target_only: bool,
num_speculative_steps: int = 1,
deduplicate_contexts: WatermarkContextScope = "none",
deduplicate_contexts_max_history: int | None = 8192,
) -> DraftWatermarker | None:
if isinstance(watermarker, SupportsSpeculativeDecoding):
return DraftWatermarker(watermarker.draft_watermarker, max_num_reqs, device)
return DraftWatermarker(
watermarker.draft_watermarker,
max_num_reqs,
device,
num_speculative_steps,
deduplicate_contexts,
deduplicate_contexts_max_history,
)
if allow_target_only:
return None
raise ValueError(
@@ -130,9 +230,10 @@ def watermarked_rejection_sample(
contexts: torch.Tensor,
watermarking: torch.Tensor,
watermarker: Watermarker,
watermarking_skip_mask: torch.Tensor | None = None,
use_fp64: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Use the target key for rejection recovery and bonus tokens."""
"""Use the target key except at contexts selected for ordinary sampling."""
assert contexts.shape == (target_logits.shape[0], watermarker.context_width)
return rejection_sample(
target_logits,
@@ -149,5 +250,6 @@ def watermarked_rejection_sample(
use_fp64=use_fp64,
contexts=contexts,
watermarking=watermarking,
watermarking_skip_mask=watermarking_skip_mask,
watermark_key=_resolve_watermark_key(watermarker),
)
+8 -1
View File
@@ -23,6 +23,10 @@ class RandomSampler:
seeds: torch.Tensor
positions: torch.Tensor
use_fp64: bool = False
is_drafting: bool = False
logits_cache: torch.Tensor | None = None
logits_cache_col: torch.Tensor | None = None
logits_cache_source: torch.Tensor | None = None
def __call__(self, logits: torch.Tensor) -> torch.Tensor:
return gumbel_sample(
@@ -32,7 +36,10 @@ class RandomSampler:
self.seeds,
self.positions,
apply_temperature=False,
is_drafting=False,
is_drafting=self.is_drafting,
logits_cache=self.logits_cache,
logits_cache_col=self.logits_cache_col,
logits_cache_source=self.logits_cache_source,
use_fp64=self.use_fp64,
)
+2 -4
View File
@@ -914,8 +914,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
spec_hidden_states = pre_hc_hidden_states[: hidden_states.shape[0]] # type: ignore[union-attr]
if isinstance(self.sampler, GPUWatermarkSampler):
self.speculator.prepare_watermarking(
self.sampler._get_contexts(input_batch.idx_mapping),
self.sampler.watermarking.gpu[input_batch.idx_mapping],
self.sampler, input_batch.idx_mapping
)
with use_workspace_lane(self._draft_workspace_lane):
self.speculator.propose(
@@ -2194,8 +2193,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
spec_hidden_states = pre_hc_hidden_states[: draft_hidden_states.size(0)]
if isinstance(self.sampler, GPUWatermarkSampler):
self.speculator.prepare_watermarking(
self.sampler._get_contexts(input_batch.idx_mapping),
self.sampler.watermarking.gpu[input_batch.idx_mapping],
self.sampler, input_batch.idx_mapping
)
with use_workspace_lane(self._draft_workspace_lane):
draft_tokens = self.speculator.propose(
+29 -1
View File
@@ -220,6 +220,8 @@ def gumbel_block_argmax(
logits_cache_stride_0,
logits_cache_stride_1,
logits_cache_col_ptr,
logits_cache_source_ptr,
logits_cache_source_stride,
vocab_size,
IS_DRAFTING: tl.constexpr,
APPLY_TEMPERATURE: tl.constexpr,
@@ -240,12 +242,16 @@ def gumbel_block_argmax(
col = tl.load(logits_cache_col_ptr + token_idx)
else:
col = tl.load(logits_cache_col_ptr)
cached_logits = tl.load(
logits_cache_source_ptr + token_idx * logits_cache_source_stride + block,
mask=mask,
)
tl.store(
logits_cache_ptr
+ req_state_idx * logits_cache_stride_0
+ col * logits_cache_stride_1
+ block,
logits,
cached_logits,
mask=mask & is_valid_req,
)
@@ -275,6 +281,8 @@ def _gumbel_sample_kernel(
logits_cache_stride_0,
logits_cache_stride_1,
logits_cache_col_ptr,
logits_cache_source_ptr,
logits_cache_source_stride,
logits_ptr,
logits_stride,
expanded_idx_mapping_ptr,
@@ -312,6 +320,8 @@ def _gumbel_sample_kernel(
logits_cache_stride_0,
logits_cache_stride_1,
logits_cache_col_ptr,
logits_cache_source_ptr,
logits_cache_source_stride,
vocab_size,
IS_DRAFTING=IS_DRAFTING,
APPLY_TEMPERATURE=APPLY_TEMPERATURE,
@@ -334,6 +344,7 @@ def gumbel_sample(
logits_cache: torch.Tensor | None = None, # [max_num_reqs, num_cols, vocab_size]
logits_cache_col: torch.Tensor | None = None, # scalar or [num_tokens]
use_fp64: bool = False,
logits_cache_source: torch.Tensor | None = None,
) -> torch.Tensor:
# Enforce contiguity on non-strided input tensors
expanded_idx_mapping = expanded_idx_mapping.contiguous()
@@ -342,11 +353,26 @@ def gumbel_sample(
logits_cache_col = logits_cache_col.contiguous()
num_tokens, vocab_size = logits.shape
if logits_cache is not None:
if logits_cache_source is None:
logits_cache_source = logits
assert logits_cache_source.shape == logits.shape, (
"logits cache source must match sampled logits shape"
)
assert logits_cache_source.device == logits.device, (
"logits cache source must be on the sampled logits device"
)
assert logits_cache_source.dtype == logits_cache.dtype, (
"logits cache source and destination must have the same dtype"
)
assert logits_cache.size(-1) >= vocab_size, (
f"draft logits cache vocab dim ({logits_cache.size(-1)}) is narrower "
f"than the sampled logits ({vocab_size}). Cached logits would be "
"truncated."
)
elif logits_cache_source is not None:
raise ValueError("logits_cache_source requires logits_cache")
if logits_cache_source is not None and logits_cache_source.stride(-1) != 1:
logits_cache_source = logits_cache_source.contiguous()
BLOCK_SIZE = 1024
num_blocks = triton.cdiv(vocab_size, BLOCK_SIZE)
local_argmax = logits.new_empty(num_tokens, num_blocks, dtype=torch.int64)
@@ -362,6 +388,8 @@ def gumbel_sample(
logits_cache.stride(0) if logits_cache is not None else 0,
logits_cache.stride(1) if logits_cache is not None else 0,
logits_cache_col,
logits_cache_source,
logits_cache_source.stride(0) if logits_cache_source is not None else 0,
logits,
logits.stride(0),
expanded_idx_mapping,
+315 -7
View File
@@ -51,12 +51,25 @@ def _repeated_context_mask_kernel(
req_indices_ptr,
prompt_lens_ptr,
total_lens_ptr,
history_offsets_ptr,
history_offsets_stride,
contexts_ptr,
context_stride,
local_positions_ptr,
prior_contexts_ptr,
prior_contexts_stride_0,
prior_contexts_stride_1,
steps_ptr,
steps_stride,
enabled_ptr,
CONTEXT_WIDTH: tl.constexpr,
NUM_SPECULATIVE_STEPS: tl.constexpr,
MAX_HISTORY: tl.constexpr,
INCLUDE_PROMPT: tl.constexpr,
SKIP_PARTIAL_CONTEXT: tl.constexpr,
HAS_HISTORY_OFFSETS: tl.constexpr,
SPECULATIVE_CONTEXTS: tl.constexpr,
DRAFT_CONTEXTS: tl.constexpr,
BLOCK: tl.constexpr,
):
row = tl.program_id(0).to(tl.int64)
@@ -71,12 +84,16 @@ def _repeated_context_mask_kernel(
return
sequence_start = 0 if INCLUDE_PROMPT else prompt_len
history_len = total_len - sequence_start
history_offset = 0
if HAS_HISTORY_OFFSETS:
history_offset = tl.load(history_offsets_ptr + row * history_offsets_stride)
offsets = tl.arange(0, BLOCK)
repeated = tl.full((), 0, tl.int32)
scan_start = 0
if MAX_HISTORY > 0:
scan_start = tl.maximum(history_len - MAX_HISTORY, 0)
remaining_history = tl.maximum(MAX_HISTORY - history_offset, 0)
scan_start = tl.maximum(history_len - remaining_history, 0)
# Aligned loop bounds let the per-lane offsets fold into immediates.
aligned_start = (scan_start // BLOCK) * BLOCK
for block_start in tl.range(aligned_start, history_len, BLOCK):
@@ -96,6 +113,93 @@ def _repeated_context_mask_kernel(
)
matches &= historical_token == context_token
repeated |= tl.max(matches.to(tl.int32), axis=0)
if SPECULATIVE_CONTEXTS:
local_pos = tl.load(local_positions_ptr + row)
if INCLUDE_PROMPT:
partial_context = tl.full((), 0, tl.int32)
for context_offset in range(CONTEXT_WIDTH):
context_token = tl.load(
contexts_ptr + row * context_stride + context_offset
)
partial_context |= context_token < 0
repeated |= partial_context
for offset in range(1, NUM_SPECULATIVE_STEPS + 1):
prior_row = row - offset
in_history = (prior_row >= 0) & (local_pos >= offset)
if MAX_HISTORY > 0:
in_history &= offset <= MAX_HISTORY
prior_req_idx = tl.load(
req_indices_ptr + tl.maximum(prior_row, 0),
mask=in_history,
other=-1,
)
prior_local_pos = tl.load(
local_positions_ptr + tl.maximum(prior_row, 0),
mask=in_history,
other=-1,
)
matches = (
in_history
& (req_idx == prior_req_idx)
& (local_pos == prior_local_pos + offset)
)
for context_offset in range(CONTEXT_WIDTH):
context_token = tl.load(
contexts_ptr + row * context_stride + context_offset
)
prior_token = tl.load(
contexts_ptr
+ tl.maximum(prior_row, 0) * context_stride
+ context_offset,
mask=in_history,
other=-2,
)
matches &= context_token == prior_token
repeated |= matches
if DRAFT_CONTEXTS:
step = tl.load(steps_ptr + row * steps_stride).to(tl.int64)
partial_context = tl.full((), 0, tl.int32)
for context_offset in range(CONTEXT_WIDTH):
context_token = tl.load(
contexts_ptr + row * context_stride + context_offset
)
partial_context |= context_token < 0
if INCLUDE_PROMPT:
repeated |= partial_context
for prior_step in range(NUM_SPECULATIVE_STEPS):
in_history = prior_step < step
if MAX_HISTORY > 0:
in_history &= prior_step >= step - MAX_HISTORY
matches = in_history
for context_offset in range(CONTEXT_WIDTH):
context_token = tl.load(
contexts_ptr + row * context_stride + context_offset
)
prior_token = tl.load(
prior_contexts_ptr
+ row * prior_contexts_stride_0
+ prior_step * prior_contexts_stride_1
+ context_offset
)
matches &= context_token == prior_token
repeated |= matches
for context_offset in range(CONTEXT_WIDTH):
context_token = tl.load(
contexts_ptr + row * context_stride + context_offset
)
tl.store(
prior_contexts_ptr
+ row * prior_contexts_stride_0
+ step * prior_contexts_stride_1
+ context_offset,
context_token,
)
enabled = tl.load(enabled_ptr + row)
repeated = enabled & (repeated == 0)
tl.store(output_ptr + row, repeated)
@@ -108,6 +212,7 @@ def _repeated_context_mask_cpu(
max_history: int | None = None,
include_prompt: bool = False,
skip_partial_context: bool = False,
history_offsets: torch.Tensor | None = None,
) -> torch.Tensor:
"""Reference implementation; the parity tests check the Triton kernel against it."""
repeated = torch.zeros(len(req_indices), dtype=torch.bool)
@@ -124,8 +229,11 @@ def _repeated_context_mask_cpu(
history_tokens = all_token_ids[req_idx, sequence_start:total_len].tolist()
prefix = [-1] * contexts.shape[-1]
current_context = tuple(contexts[row].tolist())
history_offset = 0 if history_offsets is None else int(history_offsets[row])
history_start = (
max(0, len(history_tokens) - max_history) if max_history is not None else 0
max(0, len(history_tokens) - max(0, max_history - history_offset))
if max_history is not None
else 0
)
for history_pos, token_id in enumerate(history_tokens):
if history_pos >= history_start and (
@@ -146,6 +254,9 @@ def repeated_context_mask(
max_history: int | None = None,
include_prompt: bool = False,
skip_partial_context: bool = False,
history_offsets: torch.Tensor | None = None,
local_positions: torch.Tensor | None = None,
num_speculative_steps: int = 0,
) -> torch.Tensor:
"""Return, per row, whether the row's context already occurred in its history.
@@ -163,12 +274,23 @@ def repeated_context_mask(
include_prompt: Search the prompt as well as the generated tokens.
skip_partial_context: Mark contexts containing start padding so they use
ordinary sampling.
history_offsets: Number of newer, non-committed positions preceding each
context. These positions count toward `max_history`. Must be 1-D
over rows; a stride-0 broadcast row is fine.
local_positions: Position within each request's speculative block. When
provided, compare each context with earlier contexts in that block.
num_speculative_steps: Maximum number of draft tokens in the block.
"""
if max_history is not None and max_history < 1:
raise ValueError("max_history must be positive or None")
if (local_positions is None) != (num_speculative_steps == 0):
raise ValueError(
"local_positions and positive num_speculative_steps must be "
"provided together"
)
if all_token_ids.device.type == "cpu":
return _repeated_context_mask_cpu(
repeated = _repeated_context_mask_cpu(
all_token_ids,
req_indices,
prompt_lens,
@@ -177,10 +299,42 @@ def repeated_context_mask(
max_history,
include_prompt,
skip_partial_context,
history_offsets,
)
if local_positions is not None:
max_offset = num_speculative_steps + 1
if max_history is not None:
max_offset = min(max_offset, max_history + 1)
for offset in range(1, max_offset):
prior_contexts = torch.cat(
(contexts[:offset], contexts[:-offset]), dim=0
)
prior_requests = torch.cat(
(req_indices[:offset], req_indices[:-offset]), dim=0
)
prior_local_pos = torch.cat(
(local_positions[:offset], local_positions[:-offset]), dim=0
)
repeated |= (
(req_indices >= 0)
& (local_positions >= offset)
& (req_indices == prior_requests)
& (local_positions == prior_local_pos + offset)
& (contexts == prior_contexts).all(dim=-1)
)
if include_prompt:
repeated |= (contexts < 0).any(dim=-1)
return repeated
if contexts.stride(-1) != 1:
contexts = contexts.contiguous()
# Only history_offsets passes its row stride to the kernel.
req_indices = req_indices.contiguous()
if local_positions is not None:
local_positions = local_positions.contiguous()
# `prompt_lens` and `total_lens` are indexed flat by request slot, and
# `all_token_ids` takes a row stride but is read flat within the row. Those
# three are contiguous by contract; every caller owns them outright.
repeated = torch.empty(len(req_indices), dtype=torch.bool, device=contexts.device)
_repeated_context_mask_kernel[(len(req_indices),)](
repeated,
@@ -189,17 +343,104 @@ def repeated_context_mask(
req_indices,
prompt_lens,
total_lens,
history_offsets,
0 if history_offsets is None else history_offsets.stride(0),
contexts,
contexts.stride(0),
local_positions,
None,
0,
0,
None,
0,
None,
CONTEXT_WIDTH=contexts.shape[-1],
NUM_SPECULATIVE_STEPS=num_speculative_steps,
MAX_HISTORY=0 if max_history is None else max_history,
INCLUDE_PROMPT=include_prompt,
SKIP_PARTIAL_CONTEXT=skip_partial_context,
HAS_HISTORY_OFFSETS=history_offsets is not None,
SPECULATIVE_CONTEXTS=local_positions is not None,
DRAFT_CONTEXTS=False,
BLOCK=512,
)
return repeated
def draft_watermarking_mask(
all_token_ids: torch.Tensor,
req_indices: torch.Tensor,
prompt_lens: torch.Tensor,
total_lens: torch.Tensor,
prior_contexts: torch.Tensor,
contexts: torch.Tensor,
steps: torch.Tensor,
enabled: torch.Tensor,
max_history: int | None,
include_prompt: bool,
) -> torch.Tensor:
if contexts.device.type == "cpu":
repeated = repeated_context_mask(
all_token_ids,
req_indices,
prompt_lens,
total_lens,
contexts,
max_history,
include_prompt=include_prompt,
history_offsets=steps,
)
prior_steps = torch.arange(prior_contexts.shape[1]).unsqueeze(0)
in_history = prior_steps < steps.unsqueeze(1)
if max_history is not None:
in_history &= prior_steps >= steps.unsqueeze(1) - max_history
repeated |= (
(prior_contexts[: len(contexts)] == contexts.unsqueeze(1)).all(dim=-1)
& in_history
).any(dim=-1)
if include_prompt:
repeated |= (contexts < 0).any(dim=-1)
row_indices = torch.arange(len(contexts))
prior_contexts.index_put_((row_indices, steps), contexts)
return enabled & ~repeated
if contexts.stride(-1) != 1:
contexts = contexts.contiguous()
# Only `steps` passes its row stride to the kernel.
req_indices = req_indices.contiguous()
enabled = enabled.contiguous()
active = torch.empty(len(contexts), dtype=torch.bool, device=contexts.device)
_repeated_context_mask_kernel[(len(contexts),)](
active,
all_token_ids,
all_token_ids.stride(0),
req_indices,
prompt_lens,
total_lens,
steps,
steps.stride(0),
contexts,
contexts.stride(0),
None,
prior_contexts,
prior_contexts.stride(0),
prior_contexts.stride(1),
steps,
steps.stride(0),
enabled,
NUM_SPECULATIVE_STEPS=prior_contexts.shape[1],
CONTEXT_WIDTH=contexts.shape[1],
MAX_HISTORY=0 if max_history is None else max_history,
INCLUDE_PROMPT=include_prompt,
SKIP_PARTIAL_CONTEXT=False,
HAS_HISTORY_OFFSETS=True,
SPECULATIVE_CONTEXTS=False,
DRAFT_CONTEXTS=True,
BLOCK=512,
)
return active
@triton.jit
def _mulhilo32(multiplier: tl.constexpr, value):
high = tl_math.umulhi(multiplier, value)
@@ -351,28 +592,58 @@ def _philox_gumbel_kernel(
seeds_ptr,
pos_ptr,
temp_ptr,
logits_cache_ptr,
logits_cache_stride_0,
logits_cache_stride_1,
logits_cache_col_ptr,
logits_cache_source_ptr,
logits_cache_source_stride,
key_0_value,
key_1_value,
vocab_size,
CONTEXT_WIDTH: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
USE_FP64: tl.constexpr,
IS_DRAFTING: tl.constexpr,
PER_TOKEN_COL: tl.constexpr,
):
row = tl.program_id(0).to(tl.int64)
block_index = tl.program_id(1)
groups = block_index * (BLOCK_SIZE // 4) + tl.arange(0, BLOCK_SIZE // 4)
candidate = block_index * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = candidate < vocab_size
if skip_mask_ptr is not None:
req_state_idx = row
valid_req = True
if expanded_idx_mapping_ptr is not None:
req_state_idx = tl.load(expanded_idx_mapping_ptr + row).to(tl.int64)
valid_req = req_state_idx >= 0
if logits_cache_ptr is not None:
if PER_TOKEN_COL:
col = tl.load(logits_cache_col_ptr + row)
else:
col = tl.load(logits_cache_col_ptr)
cached_logits = tl.load(
logits_cache_source_ptr + row * logits_cache_source_stride + candidate,
mask=mask,
)
tl.store(
logits_cache_ptr
+ req_state_idx * logits_cache_stride_0
+ col * logits_cache_stride_1
+ candidate,
cached_logits,
mask=mask & valid_req,
)
if skip_mask_ptr is not None: # noqa: SIM102
if tl.load(skip_mask_ptr + row):
temp = tl.load(temp_ptr + req_state_idx, mask=valid_req, other=0.0).to(
tl.float32
)
seed = tl.load(seeds_ptr + req_state_idx, mask=valid_req, other=0)
pos = tl.load(pos_ptr + row)
candidate = block_index * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = candidate < vocab_size
logits_row = logits_ptr + row * logits_stride
logits = tl.load(logits_row + candidate, mask=mask, other=float("-inf")).to(
tl.float32
@@ -384,7 +655,7 @@ def _philox_gumbel_kernel(
seed,
pos,
temp,
IS_DRAFTING=False,
IS_DRAFTING=IS_DRAFTING,
USE_FP64=USE_FP64,
APPLY_TEMPERATURE=False,
)
@@ -443,12 +714,37 @@ def philox_gumbel_sample(
seeds: torch.Tensor | None = None,
positions: torch.Tensor | None = None,
use_fp64: bool = False,
is_drafting: bool = False,
logits_cache: torch.Tensor | None = None,
logits_cache_col: torch.Tensor | None = None,
logits_cache_source: torch.Tensor | None = None,
) -> torch.Tensor:
sampling_state = (expanded_idx_mapping, temperatures, seeds, positions)
if skip_mask is None and any(value is not None for value in sampling_state):
raise ValueError("sampling state requires skip_mask")
if skip_mask is not None and any(value is None for value in sampling_state):
raise ValueError("skip_mask requires complete sampling state")
if logits_cache is not None:
if logits_cache_col is None or logits_cache_source is None:
raise ValueError("logits cache requires its column and source logits")
if expanded_idx_mapping is None:
raise ValueError("logits cache requires expanded_idx_mapping")
assert logits_cache_source.shape == logits.shape, (
"logits cache source must match sampled logits shape"
)
assert logits_cache_source.device == logits.device, (
"logits cache source must be on the sampled logits device"
)
assert logits_cache_source.dtype == logits_cache.dtype, (
"logits cache source and destination must have the same dtype"
)
assert logits_cache.size(-1) >= logits.shape[-1], (
f"draft logits cache vocab dim ({logits_cache.size(-1)}) is narrower "
f"than the sampled logits ({logits.shape[-1]}). Cached logits would be "
"truncated."
)
elif logits_cache_col is not None or logits_cache_source is not None:
raise ValueError("logits cache column and source require logits_cache")
if logits.stride(-1) != 1:
logits = logits.contiguous()
@@ -460,6 +756,10 @@ def philox_gumbel_sample(
expanded_idx_mapping = expanded_idx_mapping.contiguous()
if positions is not None:
positions = positions.contiguous()
if logits_cache_col is not None:
logits_cache_col = logits_cache_col.contiguous()
if logits_cache_source is not None and logits_cache_source.stride(-1) != 1:
logits_cache_source = logits_cache_source.contiguous()
num_tokens, vocab_size = logits.shape
block_size = 1024
num_blocks = triton.cdiv(vocab_size, block_size)
@@ -483,12 +783,20 @@ def philox_gumbel_sample(
seeds,
positions,
temperatures,
logits_cache,
logits_cache.stride(0) if logits_cache is not None else 0,
logits_cache.stride(1) if logits_cache is not None else 0,
logits_cache_col,
logits_cache_source,
logits_cache_source.stride(0) if logits_cache_source is not None else 0,
key & _UINT32_MASK_VALUE,
key >> 32,
vocab_size,
CONTEXT_WIDTH=contexts.shape[-1],
BLOCK_SIZE=block_size,
USE_FP64=use_fp64,
IS_DRAFTING=is_drafting,
PER_TOKEN_COL=logits_cache_col is not None and logits_cache_col.dim() > 0,
)
max_block_index = local_max.argmax(dim=-1, keepdim=True)
return local_argmax.gather(dim=-1, index=max_block_index).view(-1)
@@ -135,7 +135,12 @@ class DSparkSpeculator(DFlashSpeculator):
# sample_pos is the predicted token's position P. Sampling keys a draw
# by the position before the sampled token, P-1.
sampled = gumbel_sample(
sampler = (
gumbel_sample
if self.draft_watermarker is None
else self.draft_watermarker.sample
)
return sampler(
logits,
idx_map,
self.temperature,
@@ -147,11 +152,6 @@ class DSparkSpeculator(DFlashSpeculator):
logits_cache_col=self._step_cols[step],
use_fp64=self.use_fp64_gumbel,
)
if self.draft_watermarker is not None:
sampled = self.draft_watermarker.sample(
logits, sampled, idx_map, self.temperature
)
return sampled
def _sample_sequential(self, num_reqs: int, head_hidden: torch.Tensor) -> None:
if self._draft_topk is not None:
@@ -226,11 +226,18 @@ class RejectionSampler:
if self.watermark_key is None:
return {}
assert isinstance(self.sampler, GPUWatermarkSampler)
contexts = self.sampler._get_contexts(
expanded_idx_mapping, expanded_local_pos, draft_sampled
)
watermarking_skip_mask = None
if self.sampler.deduplicate_contexts != "none":
watermarking_skip_mask = self.sampler._get_repeated_contexts(
expanded_idx_mapping, contexts, expanded_local_pos
)
return {
"contexts": self.sampler._get_contexts(
expanded_idx_mapping, expanded_local_pos, draft_sampled
),
"contexts": contexts,
"watermarking": self.sampler.watermarking.gpu,
"watermarking_skip_mask": watermarking_skip_mask,
"watermark_key": self.watermark_key,
}
@@ -564,7 +564,6 @@ def _rejection_kernel(
# stores the target argmax upon first rejection, so it rejects the
# placeholder via `accepted` instead.
verifying &= is_valid_draft
if verifying:
pos = tl.load(pos_ptr + logit_idx)
u = tl_rand32(seed, pos, includes_zero=False)
@@ -719,6 +718,8 @@ def _seeded_resample_argmax(
0, # logits_cache_stride_0
0, # logits_cache_stride_1
None, # logits_cache_col_ptr
None, # logits_cache_source_ptr
0, # logits_cache_source_stride
vocab_size,
IS_DRAFTING=False,
APPLY_TEMPERATURE=False,
@@ -766,6 +767,7 @@ def _resample_kernel(
contexts_stride,
# [max_num_reqs], uint8 view of a bool tensor
watermarking_ptr,
watermarking_skip_mask_ptr,
watermark_key_0,
watermark_key_1,
vocab_size,
@@ -775,6 +777,7 @@ def _resample_kernel(
USE_BLOCK_VERIFICATION: tl.constexpr,
CONTEXT_WIDTH: tl.constexpr,
WATERMARK: tl.constexpr,
DEDUPLICATE_CONTEXTS: tl.constexpr,
):
req_idx = tl.program_id(0)
resample_idx = tl.load(rejected_step_ptr + req_idx)
@@ -807,6 +810,25 @@ def _resample_kernel(
).to(tl.float32)
# Compute the residual logits to resample the rejected token from.
is_watermarked = False
if WATERMARK:
is_watermarked = (
tl.load(
watermarking_ptr + req_state_idx,
mask=req_state_idx >= 0,
other=0,
)
!= 0
)
skip_watermarking = False
if DEDUPLICATE_CONTEXTS:
skip_watermarking = (
(tl.load(watermarking_skip_mask_ptr + resample_token_idx) != 0)
& is_watermarked
& (temp != 0.0)
)
if is_bonus or not is_valid_rejected_draft:
# Bonus token (no rejections) or -1 placeholder token. In either case,
# directly use the target logits.
@@ -868,15 +890,7 @@ def _resample_kernel(
# Resample the rejected/bonus token.
if WATERMARK:
# Padded and greedy rows retain the stock draw.
is_watermarked = (
tl.load(
watermarking_ptr + req_state_idx,
mask=req_state_idx >= 0,
other=0,
)
!= 0
)
if is_watermarked & (temp != 0.0):
if is_watermarked & (temp != 0.0) & ~skip_watermarking:
watermark_value, idx = philox_gumbel_block_argmax(
residual_logits,
mask,
@@ -1023,6 +1037,7 @@ def rejection_sample(
use_block_verification: bool = False,
contexts: torch.Tensor | None = None,
watermarking: torch.Tensor | None = None,
watermarking_skip_mask: torch.Tensor | None = None,
watermark_key: int | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
assert target_logits.ndim == 2 and target_logits.stride(-1) == 1
@@ -1036,9 +1051,13 @@ def rejection_sample(
assert watermark == (watermarking is not None) == (watermark_key is not None), (
"contexts, watermarking and watermark_key must be set together."
)
assert watermarking_skip_mask is None or watermark, (
"watermarking_skip_mask requires watermarking."
)
contexts_stride = 0
context_width = 1
watermarking_bytes: torch.Tensor | None = None
watermarking_skip_mask_bytes: torch.Tensor | None = None
watermark_key_0 = 0
watermark_key_1 = 0
if contexts is not None:
@@ -1053,6 +1072,12 @@ def rejection_sample(
contexts_stride = contexts.stride(0)
context_width = contexts.shape[-1]
watermarking_bytes = watermarking.view(torch.uint8)
if watermarking_skip_mask is not None:
assert watermarking_skip_mask.shape == (num_logits,)
assert watermarking_skip_mask.dtype == torch.bool
if not watermarking_skip_mask.is_contiguous():
watermarking_skip_mask = watermarking_skip_mask.contiguous()
watermarking_skip_mask_bytes = watermarking_skip_mask.view(torch.uint8)
watermark_key_0 = watermark_key & 0xFFFFFFFF
watermark_key_1 = watermark_key >> 32
draft_logits_stride_0 = 0
@@ -1269,6 +1294,7 @@ def rejection_sample(
contexts,
contexts_stride,
watermarking_bytes,
watermarking_skip_mask_bytes,
watermark_key_0,
watermark_key_1,
vocab_size,
@@ -1278,6 +1304,7 @@ def rejection_sample(
USE_BLOCK_VERIFICATION=use_block_verification,
CONTEXT_WIDTH=context_width,
WATERMARK=watermark,
DEDUPLICATE_CONTEXTS=watermarking_skip_mask is not None,
)
# Insert the resampled tokens into the output sampled.
+20 -7
View File
@@ -38,6 +38,7 @@ from vllm.v1.worker.gpu.spec_decode.acceptance_estimator import (
from vllm.v1.worker.utils import AttentionGroup
if TYPE_CHECKING:
from vllm.v1.watermarking.gpu_sampler import GPUWatermarkSampler
from vllm.v1.worker.gpu.pcp_manager import PCPManager
logger = init_logger(__name__)
@@ -198,6 +199,9 @@ class DraftModelSpeculator(BaseSpeculator):
self.max_num_reqs,
device,
watermark_config.allow_target_only_watermarking,
self.num_speculative_steps,
watermark_config.deduplicate_contexts,
watermark_config.deduplicate_contexts_max_history,
)
self.supports_mm_inputs = False
@@ -411,7 +415,12 @@ class DraftModelSpeculator(BaseSpeculator):
) -> torch.Tensor:
if draft_logits is not None:
logits = self.model.compute_logits(hidden_states)
sampled = gumbel_sample(
sampler = (
gumbel_sample
if self.draft_watermarker is None
else self.draft_watermarker.sample
)
sampled = sampler(
logits,
idx_mapping,
temperature,
@@ -423,10 +432,6 @@ class DraftModelSpeculator(BaseSpeculator):
logits_cache_col=draft_step,
use_fp64=self.use_fp64_gumbel,
)
if self.draft_watermarker is not None:
sampled = self.draft_watermarker.sample(
logits, sampled, idx_mapping, temperature
)
elif self.use_local_argmax_reduction:
return self.model.get_top_tokens(hidden_states)
else:
@@ -462,11 +467,19 @@ class DraftModelSpeculator(BaseSpeculator):
self.acceptance_estimator.step(idx_mapping, num_sampled, num_rejected)
def prepare_watermarking(
self, contexts: torch.Tensor, watermarking: torch.Tensor
self,
sampler: "GPUWatermarkSampler",
idx_mapping: torch.Tensor,
) -> None:
if self.draft_watermarker is None:
return
self.draft_watermarker.prepare(contexts, watermarking)
self.draft_watermarker.prepare(
sampler._get_contexts(idx_mapping),
sampler.watermarking.gpu[idx_mapping],
sampler.req_states.all_token_ids.gpu,
sampler.req_states.prompt_len.gpu,
sampler.req_states.total_len.gpu,
)
def _copy_request_inputs(
self,