[watermarking] Dual-key gumbel-max watermarking for speculative decoding support (#56122)
Signed-off-by: Raphael Rialland <raphael.rialland@mistral.ai> Signed-off-by: Simon Veitner <sveitner@redhat.com> Signed-off-by: Tomas Ruiz <tomas.ruiz.te@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Simon Veitner <sveitner@redhat.com> Co-authored-by: Tomas Ruiz <tomas.ruiz.te@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
Simon Veitner
Tomas Ruiz
parent
1ee4be4dbc
commit
7ee8a6dd01
@@ -44,6 +44,9 @@ from vllm.model_executor.warmup.replayssm_warmup import (
|
||||
from vllm.model_executor.warmup.spec_decode_rejection_warmup import (
|
||||
spec_decode_rejection_warmup,
|
||||
)
|
||||
from vllm.model_executor.warmup.watermark_sample_warmup import (
|
||||
watermark_sample_warmup,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
@@ -207,6 +210,7 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
|
||||
if enable_jit_warmup:
|
||||
kimi_k3_triton_warmup(worker)
|
||||
spec_decode_rejection_warmup(worker)
|
||||
watermark_sample_warmup(worker)
|
||||
qwen4_exp_qsa_triton_warmup(worker)
|
||||
|
||||
if enable_jit_warmup and current_platform.is_device_capability_family(100):
|
||||
|
||||
@@ -1,17 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Warm up spec-decode rejection-sampler Triton kernels.
|
||||
|
||||
The rejection sampler kernels (``_compute_local_logits_stats_kernel``,
|
||||
``_rejection_kernel``, ``_resample_kernel``) are JIT-compiled by Triton on
|
||||
first use. Without warmup, the first spec-decode request pays a multi-second
|
||||
compilation cost. This pre-compiles them with dummy data matching the
|
||||
server's vocab size and speculative config.
|
||||
"""
|
||||
"""Warm up speculative-decoding rejection kernels."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
@@ -39,42 +32,76 @@ def spec_decode_rejection_warmup(worker: Worker) -> None:
|
||||
if num_spec <= 0 or vocab_size <= 0:
|
||||
return
|
||||
|
||||
# Mirror the constexpr-relevant flags the runtime uses.
|
||||
rejection_method = getattr(spec_config, "rejection_sample_method", None)
|
||||
use_block_verification = rejection_method == "block"
|
||||
use_synthetic = rejection_method == "synthetic"
|
||||
# HAS_DRAFT_LOGITS is a constexpr, so the warmup must match the runtime.
|
||||
use_draft_logits = (
|
||||
getattr(spec_config, "draft_sample_method", "greedy") == "probabilistic"
|
||||
)
|
||||
|
||||
device = worker.device
|
||||
num_reqs = 1
|
||||
tokens_per_req = num_spec + 1
|
||||
num_logits = num_reqs * tokens_per_req
|
||||
|
||||
# Triton JIT-specializes on tensor dtypes. The target logits may be fp32
|
||||
# (apply_sampling_params copies to fp32 when processing is needed) or the
|
||||
# model dtype (pass-through otherwise), while draft logits are always the
|
||||
# model dtype. Warm every (target, draft) combination the runtime can hit.
|
||||
watermark_config = getattr(worker.vllm_config, "watermark_config", None)
|
||||
watermark_kwargs: dict[str, Any] = {}
|
||||
if watermark_config is not None:
|
||||
from vllm.v1.watermarking.spec_decode import speculative_target_watermark_key
|
||||
|
||||
try:
|
||||
watermark_key = speculative_target_watermark_key(watermark_config)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Skipping spec-decode rejection sampler warmup: could not derive "
|
||||
"the watermark recovery key.",
|
||||
exc_info=True,
|
||||
)
|
||||
return
|
||||
# Tensor dtypes and integer widths affect Triton specialization.
|
||||
watermark_kwargs = {
|
||||
"contexts": torch.zeros(
|
||||
(num_logits, watermark_config.context_width),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
),
|
||||
"watermarking": torch.zeros(num_reqs, dtype=torch.bool, device=device),
|
||||
"watermark_key": watermark_key,
|
||||
}
|
||||
|
||||
# Sampling-parameter processing may promote either distribution to fp32.
|
||||
model_dtype = model_config.dtype
|
||||
warmup_dtype_pairs = {
|
||||
(model_dtype, model_dtype),
|
||||
(torch.float32, torch.float32),
|
||||
(torch.float32, model_dtype),
|
||||
(model_dtype, torch.float32),
|
||||
}
|
||||
warmup_dtype_pairs: set[tuple[torch.dtype, torch.dtype | None]]
|
||||
if use_draft_logits:
|
||||
warmup_dtype_pairs = {
|
||||
(model_dtype, model_dtype),
|
||||
(torch.float32, torch.float32),
|
||||
(torch.float32, model_dtype),
|
||||
(model_dtype, torch.float32),
|
||||
}
|
||||
else:
|
||||
warmup_dtype_pairs = {(model_dtype, None), (torch.float32, None)}
|
||||
|
||||
logger.info(
|
||||
"Warming up spec-decode rejection sampler kernels "
|
||||
"(vocab=%d, num_spec=%d, dtype_pairs=%s, block_verify=%s).",
|
||||
"(vocab=%d, num_spec=%d, dtype_pairs=%s, block_verify=%s, watermark=%s).",
|
||||
vocab_size,
|
||||
num_spec,
|
||||
[(str(t), str(d)) for t, d in warmup_dtype_pairs],
|
||||
use_block_verification,
|
||||
watermark_config is not None,
|
||||
)
|
||||
for tgt_dtype, draft_dtype in warmup_dtype_pairs:
|
||||
target_logits = torch.zeros(
|
||||
(num_logits, vocab_size), dtype=tgt_dtype, device=device
|
||||
)
|
||||
draft_logits = torch.zeros(
|
||||
(num_reqs, num_spec, vocab_size), dtype=draft_dtype, device=device
|
||||
draft_logits = (
|
||||
torch.zeros(
|
||||
(num_reqs, num_spec, vocab_size), dtype=draft_dtype, device=device
|
||||
)
|
||||
if draft_dtype is not None
|
||||
else None
|
||||
)
|
||||
synthetic_rates = (
|
||||
torch.full((num_spec,), 0.5, dtype=torch.float32, device=device)
|
||||
@@ -85,14 +112,14 @@ def spec_decode_rejection_warmup(worker: Worker) -> None:
|
||||
rejection_sample(
|
||||
target_logits=target_logits,
|
||||
draft_logits=draft_logits,
|
||||
draft_sampled=torch.zeros(num_logits, dtype=torch.int64, device=device),
|
||||
draft_sampled=torch.zeros(num_logits, dtype=torch.int32, device=device),
|
||||
cu_num_logits=torch.tensor(
|
||||
[0, num_logits], dtype=torch.int32, device=device
|
||||
),
|
||||
pos=torch.zeros(num_logits, dtype=torch.int64, device=device),
|
||||
idx_mapping=torch.zeros(num_reqs, dtype=torch.int32, device=device),
|
||||
idx_mapping=torch.zeros(num_reqs, dtype=torch.int64, device=device),
|
||||
expanded_idx_mapping=torch.zeros(
|
||||
num_logits, dtype=torch.int32, device=device
|
||||
num_logits, dtype=torch.int64, device=device
|
||||
),
|
||||
expanded_local_pos=torch.arange(
|
||||
num_logits, dtype=torch.int32, device=device
|
||||
@@ -103,6 +130,7 @@ def spec_decode_rejection_warmup(worker: Worker) -> None:
|
||||
synthetic_conditional_rates=synthetic_rates,
|
||||
use_fp64=False,
|
||||
use_block_verification=use_block_verification,
|
||||
**watermark_kwargs,
|
||||
)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Warm up the watermarked sampler's Triton kernel.
|
||||
|
||||
``_philox_gumbel_kernel`` is JIT-compiled once per (key, logits dtype,
|
||||
skip-mask, ``USE_FP64``) specialization. The generic sampler warmup samples with
|
||||
``SamplingParams.for_sampler_warmup()``, whose feature-heavy logits processing
|
||||
forces the fp32 copy in ``apply_sampling_params``, so the first ordinary
|
||||
temperature-1.0 request is the first launch with model-dtype logits and pays
|
||||
the compilation inside inference. This pre-compiles every specialization the
|
||||
sampler path can launch for the configured watermark.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.watermarking.watermarker import Watermarker
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_NUM_WARMUP_TOKENS = 1
|
||||
|
||||
|
||||
def _philox_key(watermarker: Watermarker) -> int | None:
|
||||
"""None when the watermarker takes the torch path and compiles no kernel."""
|
||||
from vllm.v1.watermarking.gumbel import GumbelWatermarker
|
||||
from vllm.v1.watermarking.prfs import PhiloxPRF
|
||||
|
||||
if not isinstance(watermarker, GumbelWatermarker):
|
||||
return None
|
||||
prf = watermarker.prf
|
||||
return prf.key if type(prf) is PhiloxPRF else None
|
||||
|
||||
|
||||
def _philox_sampler_keys(watermarker: Watermarker) -> list[int]:
|
||||
"""Philox keys the sampler launches the kernel with."""
|
||||
from vllm.v1.watermarking.gumbel import DualKeyGumbelWatermarker
|
||||
|
||||
watermarkers: list[Watermarker] = [watermarker]
|
||||
if isinstance(watermarker, DualKeyGumbelWatermarker):
|
||||
# ``DualKeyGumbelWatermarker.sample`` launches once per key, and
|
||||
# collapses to a single key at the alpha bounds, where the other key's
|
||||
# specialization is never launched.
|
||||
if watermarker.alpha == 0:
|
||||
watermarkers = [watermarker.draft_watermarker]
|
||||
elif watermarker.alpha == 1:
|
||||
watermarkers = [watermarker.target_watermarker]
|
||||
else:
|
||||
watermarkers = [
|
||||
watermarker.draft_watermarker,
|
||||
watermarker.target_watermarker,
|
||||
]
|
||||
return [key for key in map(_philox_key, watermarkers) if key is not None]
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def watermark_sample_warmup(worker: Worker) -> None:
|
||||
if getattr(worker.vllm_config, "watermark_config", None) is None:
|
||||
return
|
||||
# ``GumbelWatermarker`` launches the kernel only on CUDA
|
||||
# (vllm/v1/watermarking/gumbel.py:63,:82); elsewhere the sampler takes the
|
||||
# torch path and warming here would compile a kernel nothing launches.
|
||||
if not current_platform.is_cuda_alike():
|
||||
return
|
||||
|
||||
from vllm.v1.watermarking.gpu_sampler import GPUWatermarkSampler
|
||||
from vllm.v1.worker.gpu.sample.watermark import philox_gumbel_sample
|
||||
|
||||
# Read the sampler the runner built instead of rebuilding it from config:
|
||||
# its watermarker (the target role under a speculative config), its context
|
||||
# deduplication setting and its fp64 flag are what the launches have to
|
||||
# match, and cannot drift from the runner. A rank that does not sample -- a
|
||||
# non-final pipeline stage, a pooling model -- holds no such sampler.
|
||||
sampler = getattr(worker.model_runner, "sampler", None)
|
||||
if not isinstance(sampler, GPUWatermarkSampler):
|
||||
return
|
||||
|
||||
model_config = worker.vllm_config.model_config
|
||||
device = worker.device
|
||||
try:
|
||||
keys = _philox_sampler_keys(sampler.watermarker)
|
||||
if not keys:
|
||||
return
|
||||
|
||||
# The sampler passes model-dtype logits through when no logits
|
||||
# processing is needed and fp32 otherwise. ``ModelConfig.dtype`` is
|
||||
# resolved to a torch dtype during validation.
|
||||
dtypes = {cast("torch.dtype", model_config.dtype), torch.float32}
|
||||
# ``GPUWatermarkSampler._sample_random`` builds a skip mask whenever
|
||||
# context deduplication is on; with it off only a batch that mixes in an
|
||||
# opted-out or greedy row does, and the mask-free specialization stays
|
||||
# reachable.
|
||||
with_skip_mask = (
|
||||
(True,) if sampler.deduplicate_contexts != "none" else (True, False)
|
||||
)
|
||||
|
||||
vocab_size = model_config.get_vocab_size()
|
||||
contexts = torch.zeros(
|
||||
(_NUM_WARMUP_TOKENS, sampler.watermarker.context_width),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
# Dtypes mirror the runtime buffers the sampler passes and are part of
|
||||
# the specialization key: int32 request-state token ids, int64 index and
|
||||
# seed buffers, fp32 sampling temperatures.
|
||||
sampling_state = {
|
||||
"skip_mask": torch.zeros(
|
||||
_NUM_WARMUP_TOKENS, dtype=torch.bool, device=device
|
||||
),
|
||||
"expanded_idx_mapping": torch.zeros(
|
||||
_NUM_WARMUP_TOKENS, dtype=torch.int64, device=device
|
||||
),
|
||||
"temperatures": torch.ones(
|
||||
_NUM_WARMUP_TOKENS, dtype=torch.float32, device=device
|
||||
),
|
||||
"seeds": torch.zeros(_NUM_WARMUP_TOKENS, dtype=torch.int64, device=device),
|
||||
"positions": torch.zeros(
|
||||
_NUM_WARMUP_TOKENS, dtype=torch.int64, device=device
|
||||
),
|
||||
}
|
||||
|
||||
logger.info(
|
||||
"Warming up watermark sampler kernel (vocab=%d, keys=%d, dtypes=%s, "
|
||||
"skip_mask=%s).",
|
||||
vocab_size,
|
||||
len(keys),
|
||||
[str(dtype) for dtype in dtypes],
|
||||
list(with_skip_mask),
|
||||
)
|
||||
for dtype in dtypes:
|
||||
logits = torch.zeros(
|
||||
(_NUM_WARMUP_TOKENS, vocab_size), dtype=dtype, device=device
|
||||
)
|
||||
for key in keys:
|
||||
for use_skip_mask in with_skip_mask:
|
||||
philox_gumbel_sample(
|
||||
logits,
|
||||
contexts,
|
||||
key,
|
||||
# Only the mixed path forwards the sampler's setting;
|
||||
# ``GumbelWatermarker._sample_watermarked`` launches
|
||||
# the mask-free variant with the fp32 default.
|
||||
use_fp64=use_skip_mask and sampler.use_fp64_gumbel,
|
||||
**(sampling_state if use_skip_mask else {}),
|
||||
)
|
||||
except Exception:
|
||||
logger.warning("Skipping watermark sampler warmup.", exc_info=True)
|
||||
Reference in New Issue
Block a user