[6/N][warmup][DSv4] Migrate sampling, and DFlash JIT kernels (#56323)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: LopezCastroRoberto <rocastro@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Roberto L. Castro
2026-09-12 17:24:24 -07:00
committed by GitHub
co-authored by Lucas Wilkinson OpenAI Codex
parent e19a3e172e
commit aed894c190
27 changed files with 2288 additions and 558 deletions
+20 -1
View File
@@ -13,11 +13,27 @@ from vllm.platforms import CpuArchEnum, current_platform
from vllm.triton_utils import HAS_TRITON
if HAS_TRITON:
from vllm.v1.sample.ops.topk_topp_triton import apply_top_k_top_p_triton
from vllm.v1.sample.ops.topk_topp_triton import (
_topk_topp,
_topp_split_mask,
_topp_split_stats,
_topp_split_step,
apply_top_k_top_p_triton,
)
logger = init_logger(__name__)
def register_top_k_top_p_warmups() -> None:
"""Register every native accelerator sampling kernel used at runtime."""
if HAS_TRITON and not current_platform.is_cpu():
_topk_topp.register_warmup()
if current_platform.is_cuda_alike():
_topp_split_stats.register_warmup()
_topp_split_step.register_warmup()
_topp_split_mask.register_warmup()
def _skip_aiter_sampler_on_gfx1250() -> bool:
# Lazy ROCm-only import; keeps arch detection out of import time on CUDA/CPU.
from vllm.platforms.rocm import on_gfx1250
@@ -128,6 +144,9 @@ class TopKTopPSampler(nn.Module):
else:
self.forward = self.forward_native
# Every accelerator backend can fall back to native sampling at runtime.
register_top_k_top_p_warmups()
def forward_native(
self,
logits: torch.Tensor,
+267 -66
View File
@@ -9,8 +9,21 @@ using Pivot-based Truncation and Selection" By Park et al.
"""
from typing import Any
import torch
from vllm.model_executor.warmup.jit_warmup import (
WarmupChoices,
WarmupIntRange,
_when,
)
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
DispatchSpec,
TritonWarmupTensor,
triton_kernel_dispatcher_with_warmup,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.math_utils import next_power_of_2
@@ -863,6 +876,90 @@ def _topk_topp_kernel(
tl.store(LOGITS_ROW + offs_n, logits_blk, mask=mask_n)
def _max_sampler_batch_size(vllm_config: Any) -> int:
return vllm_config.scheduler_config.max_num_seqs * (
vllm_config.num_speculative_tokens + 1
)
def _topk_topp_warmup_inputs(vllm_config: Any) -> dict[str, Any]:
vocab_size = vllm_config.model_config.get_vocab_size()
split_enabled = current_platform.is_cuda_alike()
mode: Any = WarmupChoices((True, False), (True, True), (False, True))
logits_stride: Any = WarmupChoices(16, 2)
batch_size: Any = WarmupIntRange(1, _max_sampler_batch_size(vllm_config) + 1)
topk_enabled = mode[0]
topp_enabled = mode[1]
_when(topk_enabled or not (split_enabled and batch_size <= _SPLIT_MAX_BATCH))
logits = TritonWarmupTensor(
torch.float32,
shape=(batch_size, vocab_size),
strides=(logits_stride, 1),
)
return dict(
logits=logits,
buffer=TritonWarmupTensor(torch.float32),
percentile_to_std_table=TritonWarmupTensor(torch.float32),
normal_cdf_to_sigma_table=TritonWarmupTensor(torch.float32),
k=TritonWarmupTensor(torch.int32) if topk_enabled else None,
p=TritonWarmupTensor(torch.float32) if topp_enabled else None,
mask_value=float("-inf"),
num_sm=num_compute_units(),
)
@triton_kernel_dispatcher_with_warmup(
kernel=_topk_topp_kernel,
warmup_inputs=_topk_topp_warmup_inputs,
)
def _topk_topp(
logits: torch.Tensor,
buffer: torch.Tensor,
percentile_to_std_table: torch.Tensor,
normal_cdf_to_sigma_table: torch.Tensor,
k: torch.Tensor | None,
p: torch.Tensor | None,
mask_value: float,
num_sm: int,
) -> DispatchSpec:
batch_size, vocab_size = logits.shape
topk_enabled = k is not None
topp_enabled = p is not None
k_ptr = k if k is not None else logits
p_ptr = p if p is not None else logits
num_programs = min(num_sm, batch_size)
# Smaller tiles compile and run faster on CPU; GPU benefits from larger tiles.
# On XPU, large BLOCK_SIZE causes precision loss in the single-pass pivot
# approximation; use smaller tiles for accurate top-p results.
if logits.device.type == "cpu":
block_size, block_size_trunc, num_warps = 256, 128, 4
elif logits.device.type == "xpu":
block_size, block_size_trunc, num_warps = 4096, 2048, 4
else:
block_size, block_size_trunc, num_warps = 8192, 4096, 8
# Each program serially sweeps the vocab row in BLOCK_SIZE tiles, so
# per-tile latency bounds kernel latency, and Triton's default of 4
# warps leaves an 8192-wide tile at 16 elements per lane. 8 warps is
# faster on every arch measured (SM90, SM100, SM120, gfx950); 16 is not.
split_covers_ponly = (
current_platform.is_cuda_alike()
and topp_enabled
and batch_size <= _SPLIT_MAX_BATCH
)
return (num_programs,), dict(
K=k_ptr,
P=p_ptr,
BATCH_SIZE=batch_size,
VOCAB_SIZE=vocab_size,
BLOCK_SIZE=block_size,
BLOCK_SIZE_TRUNC=block_size_trunc,
TOPK_ENABLED=topk_enabled,
TOPP_ENABLED=topp_enabled,
SPLIT_COVERS_PONLY=split_covers_ponly,
num_warps=num_warps,
)
# ---------------------------------------------------------------------------
# Split-row pipeline for small batches.
#
@@ -892,6 +989,13 @@ _SPLIT_ROUNDS = 5
_TRITON_SPLIT_CACHE: dict[torch.device, dict[str, torch.Tensor]] = {}
def _topp_split_count(batch_size: int, num_sm: int) -> int:
splits = 1
while splits < _SPLIT_MAX_SPLITS and splits * 2 * batch_size <= num_sm:
splits *= 2
return splits
@triton.jit
def _topp_sb_stats_kernel(
LOGITS,
@@ -939,6 +1043,51 @@ def _topp_sb_stats_kernel(
tl.store(STATS + base + 3, cnt)
def _topp_split_stats_warmup_inputs(vllm_config: Any) -> dict[str, Any]:
vocab_size = vllm_config.model_config.get_vocab_size()
num_sm = num_compute_units()
logits_stride: Any = WarmupChoices(16, 2)
batch_size: Any = WarmupIntRange(
1, min(_max_sampler_batch_size(vllm_config), _SPLIT_MAX_BATCH) + 1
)
has_k: Any = WarmupChoices(False, True)
logits = TritonWarmupTensor(
torch.float32,
shape=(batch_size, vocab_size),
strides=(logits_stride, 1),
)
return dict(
logits=logits,
stats=TritonWarmupTensor(torch.float32),
k=TritonWarmupTensor(torch.int32) if has_k else None,
p=TritonWarmupTensor(torch.float32),
num_sm=num_sm,
)
@triton_kernel_dispatcher_with_warmup(
kernel=_topp_sb_stats_kernel,
warmup_inputs=_topp_split_stats_warmup_inputs,
)
def _topp_split_stats(
logits: torch.Tensor,
stats: torch.Tensor,
k: torch.Tensor | None,
p: torch.Tensor,
num_sm: int,
) -> DispatchSpec:
batch_size, vocab_size = logits.shape
splits = _topp_split_count(batch_size, num_sm)
return (logits.shape[0] * splits,), dict(
K=k if k is not None else logits,
HAS_K=k is not None,
VOCAB_SIZE=vocab_size,
S=splits,
BLOCK=8192,
num_warps=8,
)
@triton.jit
def _topp_sb_row_stats(STATS, row, S: tl.constexpr):
"""Combine per-slice partials into row (max, sum_exp, min, n_finite)."""
@@ -1183,6 +1332,58 @@ def _topp_sb_step_kernel(
tl.store(pbase + fidx * 3 + 2, nmin)
def _topp_split_step_warmup_inputs(vllm_config: Any) -> dict[str, Any]:
vocab_size = vllm_config.model_config.get_vocab_size()
num_sm = num_compute_units()
logits_stride: Any = WarmupChoices(16, 2)
batch_size: Any = WarmupIntRange(
1, min(_max_sampler_batch_size(vllm_config), _SPLIT_MAX_BATCH) + 1
)
round: Any = WarmupIntRange(0, _SPLIT_ROUNDS)
has_k: Any = WarmupChoices(False, True)
logits = TritonWarmupTensor(
torch.float32,
shape=(batch_size, vocab_size),
strides=(logits_stride, 1),
)
return dict(
logits=logits,
stats=TritonWarmupTensor(torch.float32),
parts=TritonWarmupTensor(torch.float32),
k=TritonWarmupTensor(torch.int32) if has_k else None,
p=TritonWarmupTensor(torch.float32),
round=round,
num_sm=num_sm,
)
@triton_kernel_dispatcher_with_warmup(
kernel=_topp_sb_step_kernel,
warmup_inputs=_topp_split_step_warmup_inputs,
)
def _topp_split_step(
logits: torch.Tensor,
stats: torch.Tensor,
parts: torch.Tensor,
k: torch.Tensor | None,
p: torch.Tensor,
round: int,
num_sm: int,
) -> DispatchSpec:
batch_size, vocab_size = logits.shape
splits = _topp_split_count(batch_size, num_sm)
return (logits.shape[0] * splits,), dict(
K=k if k is not None else logits,
HAS_K=k is not None,
S=splits,
F=_SPLIT_FANOUT,
NUM_ROUNDS=_SPLIT_ROUNDS,
VOCAB_SIZE=vocab_size,
BLOCK=2048,
num_warps=8,
)
@triton.jit
def _topp_sb_mask_kernel(
LOGITS,
@@ -1313,6 +1514,57 @@ def _topp_sb_mask_kernel(
tl.store(ROW + i + offs, tl.where(keep, x, MASK_VALUE), mask=mask_n)
def _topp_split_mask_warmup_inputs(vllm_config: Any) -> dict[str, Any]:
vocab_size = vllm_config.model_config.get_vocab_size()
num_sm = num_compute_units()
logits_stride: Any = WarmupChoices(16, 2)
batch_size: Any = WarmupIntRange(
1, min(_max_sampler_batch_size(vllm_config), _SPLIT_MAX_BATCH) + 1
)
has_k: Any = WarmupChoices(False, True)
logits = TritonWarmupTensor(
torch.float32,
shape=(batch_size, vocab_size),
strides=(logits_stride, 1),
)
return dict(
logits=logits,
stats=TritonWarmupTensor(torch.float32),
parts=TritonWarmupTensor(torch.float32),
k=TritonWarmupTensor(torch.int32) if has_k else None,
p=TritonWarmupTensor(torch.float32),
mask_value=float("-inf"),
num_sm=num_sm,
)
@triton_kernel_dispatcher_with_warmup(
kernel=_topp_sb_mask_kernel,
warmup_inputs=_topp_split_mask_warmup_inputs,
)
def _topp_split_mask(
logits: torch.Tensor,
stats: torch.Tensor,
parts: torch.Tensor,
k: torch.Tensor | None,
p: torch.Tensor,
mask_value: float,
num_sm: int,
) -> DispatchSpec:
batch_size, vocab_size = logits.shape
splits = _topp_split_count(batch_size, num_sm)
return (logits.shape[0] * splits,), dict(
K=k if k is not None else logits,
HAS_K=k is not None,
S=splits,
F=_SPLIT_FANOUT,
NUM_ROUNDS=_SPLIT_ROUNDS,
VOCAB_SIZE=vocab_size,
BLOCK=8192,
num_warps=8,
)
def _apply_topp_split(
logits: torch.Tensor,
k: torch.Tensor | None,
@@ -1321,11 +1573,6 @@ def _apply_topp_split(
num_sm: int,
) -> None:
"""Split-row top-p pipeline for small batches; masks logits in place."""
batch_size, vocab_size = logits.shape
splits = 1
while splits < _SPLIT_MAX_SPLITS and splits * 2 * batch_size <= num_sm:
splits *= 2
ws = _TRITON_SPLIT_CACHE.get(logits.device)
if ws is None:
ws = {
@@ -1336,53 +1583,31 @@ def _apply_topp_split(
}
_TRITON_SPLIT_CACHE[logits.device] = ws
k_ptr = k if k is not None else logits # dummy pointer when HAS_K=False
has_k = k is not None
_topp_sb_stats_kernel[(batch_size * splits,)](
_topp_split_stats(
logits,
logits.stride(0),
ws["stats"],
k_ptr,
k,
p,
HAS_K=has_k,
VOCAB_SIZE=vocab_size,
S=splits,
BLOCK=8192,
num_warps=8,
num_sm,
)
for round_i in range(_SPLIT_ROUNDS):
_topp_sb_step_kernel[(batch_size * splits,)](
_topp_split_step(
logits,
logits.stride(0),
ws["stats"],
ws["parts"],
k_ptr,
k,
p,
round_i,
HAS_K=has_k,
S=splits,
F=_SPLIT_FANOUT,
NUM_ROUNDS=_SPLIT_ROUNDS,
VOCAB_SIZE=vocab_size,
BLOCK=2048,
num_warps=8,
num_sm,
)
_topp_sb_mask_kernel[(batch_size * splits,)](
_topp_split_mask(
logits,
logits.stride(0),
ws["stats"],
ws["parts"],
k_ptr,
k,
p,
HAS_K=has_k,
MASK_VALUE=mask_value,
S=splits,
F=_SPLIT_FANOUT,
NUM_ROUNDS=_SPLIT_ROUNDS,
VOCAB_SIZE=vocab_size,
BLOCK=8192,
num_warps=8,
mask_value,
num_sm,
)
@@ -1477,39 +1702,15 @@ def apply_top_k_top_p_triton(
else:
normal_cdf_to_sigma_table, percentile_to_std_table = tables
# Smaller tiles compile and run faster on CPU; GPU benefits from larger tiles.
# On XPU, large BLOCK_SIZE causes precision loss in the single-pass pivot
# approximation; use smaller tiles for accurate top-p results.
launch_kwargs = {}
if logits.device.type == "cpu":
block_size, block_size_trunc = 256, 128
elif logits.device.type == "xpu":
block_size, block_size_trunc = 4096, 2048
else:
block_size, block_size_trunc = 8192, 4096
# Each program serially sweeps the vocab row in BLOCK_SIZE tiles, so
# per-tile latency bounds kernel latency, and Triton's default of 4
# warps leaves an 8192-wide tile at 16 elements per lane. 8 warps is
# faster on every arch measured (SM90, SM100, SM120, gfx950); 16 is not.
launch_kwargs["num_warps"] = 8
_topk_topp_kernel[(NUM_PROGRAMS,)](
_topk_topp(
logits,
logits.stride(0),
buffer,
percentile_to_std_table,
normal_cdf_to_sigma_table,
k_ptr,
p_ptr,
BATCH_SIZE=batch_size,
MASK_VALUE=mask_value,
VOCAB_SIZE=vocab_size,
BLOCK_SIZE=block_size,
BLOCK_SIZE_TRUNC=block_size_trunc,
TOPK_ENABLED=topk_enabled,
TOPP_ENABLED=topp_enabled,
SPLIT_COVERS_PONLY=use_split,
**launch_kwargs,
k_ptr if topk_enabled else None,
p_ptr if topp_enabled else None,
mask_value,
num_sm,
)
if use_split:
_apply_topp_split(logits, k_ptr, p_ptr, mask_value, num_sm)
+186 -15
View File
@@ -5,13 +5,19 @@ from __future__ import annotations
from collections.abc import Sequence
from dataclasses import replace
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
import torch
import torch.nn as nn
from vllm.config.model import PROCESSED_LOGPROBS_MODES
from vllm.logger import init_logger
from vllm.model_executor.warmup.jit_warmup import WarmupChoices
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
DispatchSpec,
TritonWarmupTensor,
triton_kernel_dispatcher_with_warmup,
)
from vllm.triton_utils import tl, triton
from vllm.v1.outputs import LogprobsLists, LogprobsTensors, SamplerOutput
from vllm.v1.sample.logits_processor.builtin import MinTokensLogitsProcessor
@@ -88,6 +94,14 @@ class RejectionSampler(nn.Module):
device=device,
)
self.synthetic_mode = self.synthetic_conditional_rates is not None
_rejection_greedy_sample.register_warmup(synthetic_mode=self.synthetic_mode)
_rejection_random_sample.register_warmup(
synthetic_mode=self.synthetic_mode,
)
_expand.register_warmup()
_sample_recovered_tokens.register_warmup(
use_fp64_gumbel=self.use_fp64_gumbel,
)
def forward(
self,
@@ -453,7 +467,7 @@ def rejection_sample(
if not sampling_metadata.all_random:
# Rejection sampling for greedy sampling requests.
target_argmax = target_logits.argmax(dim=-1)
rejection_greedy_sample_kernel[(batch_size,)](
_rejection_greedy_sample(
output_token_ids,
cu_num_draft_tokens,
draft_token_ids,
@@ -463,7 +477,6 @@ def rejection_sample(
max_spec_len,
uniform_probs,
synthetic_conditional_rates,
SYNTHETIC_MODE=synthetic_mode,
)
if sampling_metadata.all_greedy:
return output_token_ids
@@ -488,7 +501,7 @@ def rejection_sample(
# Rejection sampling for random sampling requests.
assert uniform_probs is not None
rejection_random_sample_kernel[(batch_size,)](
_rejection_random_sample(
output_token_ids,
cu_num_draft_tokens,
draft_token_ids,
@@ -499,10 +512,7 @@ def rejection_sample(
uniform_probs,
is_greedy,
max_spec_len,
vocab_size,
synthetic_conditional_rates,
NO_DRAFT_PROBS=draft_probs is None,
SYNTHETIC_MODE=synthetic_mode,
)
return output_token_ids
@@ -594,13 +604,12 @@ def expand_batch_to_tokens(
batch_size = x.shape[0]
assert cu_num_tokens.shape[0] == batch_size
expanded_x = x.new_empty(num_tokens)
expand_kernel[(batch_size,)](
_expand(
expanded_x,
x,
cu_num_tokens,
replace_from,
replace_to,
MAX_NUM_TOKENS=MAX_SPEC_LEN, # To avoid recompilation.
)
return expanded_x
@@ -694,18 +703,14 @@ def sample_recovered_tokens(
inv_q = q.reciprocal()
recovered_token_ids = torch.empty_like(draft_token_ids)
BLOCK_SIZE = 8192
sample_recovered_tokens_kernel[(batch_size, max_spec_len)](
_sample_recovered_tokens(
recovered_token_ids,
cu_num_draft_tokens,
draft_token_ids,
draft_probs,
target_probs,
inv_q,
vocab_size,
BLOCK_SIZE,
NO_DRAFT_PROBS=draft_probs is None,
USE_FP64_GUMBEL=use_fp64_gumbel,
max_spec_len=max_spec_len,
)
return recovered_token_ids
@@ -769,6 +774,50 @@ def rejection_greedy_sample_kernel(
)
def _rejection_greedy_sample_warmup_inputs(
*, synthetic_mode: bool
) -> dict[str, object]:
all_greedy = WarmupChoices(False, True)
int32 = TritonWarmupTensor(torch.int32)
return dict(
output_token_ids=int32,
cu_num_draft_tokens=int32,
draft_token_ids=int32,
target_argmax=TritonWarmupTensor(torch.int64),
bonus_token_ids=TritonWarmupTensor(torch.int64),
is_greedy=None if all_greedy else TritonWarmupTensor(torch.bool),
max_spec_len=5,
uniform_probs=(
None
if all_greedy and not synthetic_mode
else TritonWarmupTensor(torch.float64)
),
synthetic_conditional_rates=(
TritonWarmupTensor(torch.float32) if synthetic_mode else None
),
)
@triton_kernel_dispatcher_with_warmup(
kernel=rejection_greedy_sample_kernel,
warmup_inputs=_rejection_greedy_sample_warmup_inputs,
)
def _rejection_greedy_sample(
output_token_ids: torch.Tensor,
cu_num_draft_tokens: torch.Tensor,
draft_token_ids: torch.Tensor,
target_argmax: torch.Tensor,
bonus_token_ids: torch.Tensor,
is_greedy: torch.Tensor | None,
max_spec_len: int,
uniform_probs: torch.Tensor | None,
synthetic_conditional_rates: torch.Tensor | None,
) -> DispatchSpec:
return (output_token_ids.shape[0],), dict(
SYNTHETIC_MODE=synthetic_conditional_rates is not None,
)
# NOTE(woosuk): Avoid specialization to prevent unnecessary recompilation.
@triton.jit(do_not_specialize=["max_spec_len"])
def rejection_random_sample_kernel(
@@ -845,6 +894,57 @@ def rejection_random_sample_kernel(
)
def _rejection_random_sample_warmup_inputs(
*, synthetic_mode: bool
) -> dict[str, object]:
vocab_size: Any = WarmupChoices(2, 16)
no_draft_probs = WarmupChoices(False, True)
int32 = TritonWarmupTensor(torch.int32)
probabilities = TritonWarmupTensor(
torch.float32,
shape=(1, vocab_size),
)
return dict(
output_token_ids=int32,
cu_num_draft_tokens=int32,
draft_token_ids=int32,
draft_probs=None if no_draft_probs else probabilities,
target_probs=probabilities,
bonus_token_ids=TritonWarmupTensor(torch.int64),
recovered_token_ids=int32,
uniform_probs=TritonWarmupTensor(torch.float64),
is_greedy=TritonWarmupTensor(torch.bool),
max_spec_len=5,
synthetic_conditional_rates=(
TritonWarmupTensor(torch.float32) if synthetic_mode else None
),
)
@triton_kernel_dispatcher_with_warmup(
kernel=rejection_random_sample_kernel,
warmup_inputs=_rejection_random_sample_warmup_inputs,
)
def _rejection_random_sample(
output_token_ids: torch.Tensor,
cu_num_draft_tokens: torch.Tensor,
draft_token_ids: torch.Tensor,
draft_probs: torch.Tensor | None,
target_probs: torch.Tensor,
bonus_token_ids: torch.Tensor,
recovered_token_ids: torch.Tensor,
uniform_probs: torch.Tensor,
is_greedy: torch.Tensor,
max_spec_len: int,
synthetic_conditional_rates: torch.Tensor | None,
) -> DispatchSpec:
return (output_token_ids.shape[0],), dict(
vocab_size=target_probs.shape[1],
NO_DRAFT_PROBS=draft_probs is None,
SYNTHETIC_MODE=synthetic_conditional_rates is not None,
)
# NOTE(woosuk): Avoid specialization to prevent unnecessary recompilation.
@triton.jit(do_not_specialize=["replace_from", "replace_to"])
def expand_kernel(
@@ -869,6 +969,31 @@ def expand_kernel(
tl.store(output_ptr + start_idx + offset, src_val, mask=offset < num_tokens)
def _expand_warmup_inputs() -> dict[str, object]:
dtype = WarmupChoices(torch.float32, torch.int32, torch.int64)
data = TritonWarmupTensor(dtype)
return dict(
output=data,
input=data,
cu_num_tokens=TritonWarmupTensor(torch.int32),
replace_from=0,
replace_to=0,
)
@triton_kernel_dispatcher_with_warmup(
kernel=expand_kernel, warmup_inputs=_expand_warmup_inputs
)
def _expand(
output: torch.Tensor,
input: torch.Tensor,
cu_num_tokens: torch.Tensor,
replace_from: int,
replace_to: int,
) -> DispatchSpec:
return (input.shape[0],), dict(MAX_NUM_TOKENS=MAX_SPEC_LEN)
@triton.jit
def sample_recovered_tokens_kernel(
output_token_ids_ptr, # [num_tokens]
@@ -951,3 +1076,49 @@ def sample_recovered_tokens_kernel(
recovered_id = tl.minimum(recovered_id, vocab_size - 1)
tl.store(output_token_ids_ptr + token_idx, recovered_id)
def _sample_recovered_tokens_warmup_inputs(
*, use_fp64_gumbel: bool
) -> dict[str, object]:
vocab_size: Any = WarmupChoices(2, 16)
no_draft_probs = WarmupChoices(False, True)
int32 = TritonWarmupTensor(torch.int32)
probabilities = TritonWarmupTensor(
torch.float32,
shape=(1, vocab_size),
)
return dict(
output_token_ids=int32,
cu_num_draft_tokens=int32,
draft_token_ids=int32,
draft_probs=None if no_draft_probs else probabilities,
target_probs=probabilities,
inv_q=TritonWarmupTensor(
torch.float64 if use_fp64_gumbel else torch.float32,
shape=(1, vocab_size),
),
max_spec_len=5,
)
@triton_kernel_dispatcher_with_warmup(
kernel=sample_recovered_tokens_kernel,
warmup_inputs=_sample_recovered_tokens_warmup_inputs,
)
def _sample_recovered_tokens(
output_token_ids: torch.Tensor,
cu_num_draft_tokens: torch.Tensor,
draft_token_ids: torch.Tensor,
draft_probs: torch.Tensor | None,
target_probs: torch.Tensor,
inv_q: torch.Tensor,
*,
max_spec_len: int,
) -> DispatchSpec:
return (cu_num_draft_tokens.shape[0], max_spec_len), dict(
vocab_size=target_probs.shape[1],
BLOCK_SIZE=8192,
NO_DRAFT_PROBS=draft_probs is None,
USE_FP64_GUMBEL=inv_q.dtype == torch.float64,
)