[6/N][warmup][DSv4] Migrate sampling, and DFlash JIT kernels (#56323)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: LopezCastroRoberto <rocastro@redhat.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Lucas Wilkinson
OpenAI Codex
parent
e19a3e172e
commit
aed894c190
@@ -13,11 +13,27 @@ from vllm.platforms import CpuArchEnum, current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
|
||||
if HAS_TRITON:
|
||||
from vllm.v1.sample.ops.topk_topp_triton import apply_top_k_top_p_triton
|
||||
from vllm.v1.sample.ops.topk_topp_triton import (
|
||||
_topk_topp,
|
||||
_topp_split_mask,
|
||||
_topp_split_stats,
|
||||
_topp_split_step,
|
||||
apply_top_k_top_p_triton,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def register_top_k_top_p_warmups() -> None:
|
||||
"""Register every native accelerator sampling kernel used at runtime."""
|
||||
if HAS_TRITON and not current_platform.is_cpu():
|
||||
_topk_topp.register_warmup()
|
||||
if current_platform.is_cuda_alike():
|
||||
_topp_split_stats.register_warmup()
|
||||
_topp_split_step.register_warmup()
|
||||
_topp_split_mask.register_warmup()
|
||||
|
||||
|
||||
def _skip_aiter_sampler_on_gfx1250() -> bool:
|
||||
# Lazy ROCm-only import; keeps arch detection out of import time on CUDA/CPU.
|
||||
from vllm.platforms.rocm import on_gfx1250
|
||||
@@ -128,6 +144,9 @@ class TopKTopPSampler(nn.Module):
|
||||
else:
|
||||
self.forward = self.forward_native
|
||||
|
||||
# Every accelerator backend can fall back to native sampling at runtime.
|
||||
register_top_k_top_p_warmups()
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
|
||||
@@ -9,8 +9,21 @@ using Pivot-based Truncation and Selection" By Park et al.
|
||||
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup import (
|
||||
WarmupChoices,
|
||||
WarmupIntRange,
|
||||
_when,
|
||||
)
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
DispatchSpec,
|
||||
TritonWarmupTensor,
|
||||
triton_kernel_dispatcher_with_warmup,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
@@ -863,6 +876,90 @@ def _topk_topp_kernel(
|
||||
tl.store(LOGITS_ROW + offs_n, logits_blk, mask=mask_n)
|
||||
|
||||
|
||||
def _max_sampler_batch_size(vllm_config: Any) -> int:
|
||||
return vllm_config.scheduler_config.max_num_seqs * (
|
||||
vllm_config.num_speculative_tokens + 1
|
||||
)
|
||||
|
||||
|
||||
def _topk_topp_warmup_inputs(vllm_config: Any) -> dict[str, Any]:
|
||||
vocab_size = vllm_config.model_config.get_vocab_size()
|
||||
split_enabled = current_platform.is_cuda_alike()
|
||||
mode: Any = WarmupChoices((True, False), (True, True), (False, True))
|
||||
logits_stride: Any = WarmupChoices(16, 2)
|
||||
batch_size: Any = WarmupIntRange(1, _max_sampler_batch_size(vllm_config) + 1)
|
||||
topk_enabled = mode[0]
|
||||
topp_enabled = mode[1]
|
||||
_when(topk_enabled or not (split_enabled and batch_size <= _SPLIT_MAX_BATCH))
|
||||
logits = TritonWarmupTensor(
|
||||
torch.float32,
|
||||
shape=(batch_size, vocab_size),
|
||||
strides=(logits_stride, 1),
|
||||
)
|
||||
return dict(
|
||||
logits=logits,
|
||||
buffer=TritonWarmupTensor(torch.float32),
|
||||
percentile_to_std_table=TritonWarmupTensor(torch.float32),
|
||||
normal_cdf_to_sigma_table=TritonWarmupTensor(torch.float32),
|
||||
k=TritonWarmupTensor(torch.int32) if topk_enabled else None,
|
||||
p=TritonWarmupTensor(torch.float32) if topp_enabled else None,
|
||||
mask_value=float("-inf"),
|
||||
num_sm=num_compute_units(),
|
||||
)
|
||||
|
||||
|
||||
@triton_kernel_dispatcher_with_warmup(
|
||||
kernel=_topk_topp_kernel,
|
||||
warmup_inputs=_topk_topp_warmup_inputs,
|
||||
)
|
||||
def _topk_topp(
|
||||
logits: torch.Tensor,
|
||||
buffer: torch.Tensor,
|
||||
percentile_to_std_table: torch.Tensor,
|
||||
normal_cdf_to_sigma_table: torch.Tensor,
|
||||
k: torch.Tensor | None,
|
||||
p: torch.Tensor | None,
|
||||
mask_value: float,
|
||||
num_sm: int,
|
||||
) -> DispatchSpec:
|
||||
batch_size, vocab_size = logits.shape
|
||||
topk_enabled = k is not None
|
||||
topp_enabled = p is not None
|
||||
k_ptr = k if k is not None else logits
|
||||
p_ptr = p if p is not None else logits
|
||||
num_programs = min(num_sm, batch_size)
|
||||
# Smaller tiles compile and run faster on CPU; GPU benefits from larger tiles.
|
||||
# On XPU, large BLOCK_SIZE causes precision loss in the single-pass pivot
|
||||
# approximation; use smaller tiles for accurate top-p results.
|
||||
if logits.device.type == "cpu":
|
||||
block_size, block_size_trunc, num_warps = 256, 128, 4
|
||||
elif logits.device.type == "xpu":
|
||||
block_size, block_size_trunc, num_warps = 4096, 2048, 4
|
||||
else:
|
||||
block_size, block_size_trunc, num_warps = 8192, 4096, 8
|
||||
# Each program serially sweeps the vocab row in BLOCK_SIZE tiles, so
|
||||
# per-tile latency bounds kernel latency, and Triton's default of 4
|
||||
# warps leaves an 8192-wide tile at 16 elements per lane. 8 warps is
|
||||
# faster on every arch measured (SM90, SM100, SM120, gfx950); 16 is not.
|
||||
split_covers_ponly = (
|
||||
current_platform.is_cuda_alike()
|
||||
and topp_enabled
|
||||
and batch_size <= _SPLIT_MAX_BATCH
|
||||
)
|
||||
return (num_programs,), dict(
|
||||
K=k_ptr,
|
||||
P=p_ptr,
|
||||
BATCH_SIZE=batch_size,
|
||||
VOCAB_SIZE=vocab_size,
|
||||
BLOCK_SIZE=block_size,
|
||||
BLOCK_SIZE_TRUNC=block_size_trunc,
|
||||
TOPK_ENABLED=topk_enabled,
|
||||
TOPP_ENABLED=topp_enabled,
|
||||
SPLIT_COVERS_PONLY=split_covers_ponly,
|
||||
num_warps=num_warps,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Split-row pipeline for small batches.
|
||||
#
|
||||
@@ -892,6 +989,13 @@ _SPLIT_ROUNDS = 5
|
||||
_TRITON_SPLIT_CACHE: dict[torch.device, dict[str, torch.Tensor]] = {}
|
||||
|
||||
|
||||
def _topp_split_count(batch_size: int, num_sm: int) -> int:
|
||||
splits = 1
|
||||
while splits < _SPLIT_MAX_SPLITS and splits * 2 * batch_size <= num_sm:
|
||||
splits *= 2
|
||||
return splits
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _topp_sb_stats_kernel(
|
||||
LOGITS,
|
||||
@@ -939,6 +1043,51 @@ def _topp_sb_stats_kernel(
|
||||
tl.store(STATS + base + 3, cnt)
|
||||
|
||||
|
||||
def _topp_split_stats_warmup_inputs(vllm_config: Any) -> dict[str, Any]:
|
||||
vocab_size = vllm_config.model_config.get_vocab_size()
|
||||
num_sm = num_compute_units()
|
||||
logits_stride: Any = WarmupChoices(16, 2)
|
||||
batch_size: Any = WarmupIntRange(
|
||||
1, min(_max_sampler_batch_size(vllm_config), _SPLIT_MAX_BATCH) + 1
|
||||
)
|
||||
has_k: Any = WarmupChoices(False, True)
|
||||
logits = TritonWarmupTensor(
|
||||
torch.float32,
|
||||
shape=(batch_size, vocab_size),
|
||||
strides=(logits_stride, 1),
|
||||
)
|
||||
return dict(
|
||||
logits=logits,
|
||||
stats=TritonWarmupTensor(torch.float32),
|
||||
k=TritonWarmupTensor(torch.int32) if has_k else None,
|
||||
p=TritonWarmupTensor(torch.float32),
|
||||
num_sm=num_sm,
|
||||
)
|
||||
|
||||
|
||||
@triton_kernel_dispatcher_with_warmup(
|
||||
kernel=_topp_sb_stats_kernel,
|
||||
warmup_inputs=_topp_split_stats_warmup_inputs,
|
||||
)
|
||||
def _topp_split_stats(
|
||||
logits: torch.Tensor,
|
||||
stats: torch.Tensor,
|
||||
k: torch.Tensor | None,
|
||||
p: torch.Tensor,
|
||||
num_sm: int,
|
||||
) -> DispatchSpec:
|
||||
batch_size, vocab_size = logits.shape
|
||||
splits = _topp_split_count(batch_size, num_sm)
|
||||
return (logits.shape[0] * splits,), dict(
|
||||
K=k if k is not None else logits,
|
||||
HAS_K=k is not None,
|
||||
VOCAB_SIZE=vocab_size,
|
||||
S=splits,
|
||||
BLOCK=8192,
|
||||
num_warps=8,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _topp_sb_row_stats(STATS, row, S: tl.constexpr):
|
||||
"""Combine per-slice partials into row (max, sum_exp, min, n_finite)."""
|
||||
@@ -1183,6 +1332,58 @@ def _topp_sb_step_kernel(
|
||||
tl.store(pbase + fidx * 3 + 2, nmin)
|
||||
|
||||
|
||||
def _topp_split_step_warmup_inputs(vllm_config: Any) -> dict[str, Any]:
|
||||
vocab_size = vllm_config.model_config.get_vocab_size()
|
||||
num_sm = num_compute_units()
|
||||
logits_stride: Any = WarmupChoices(16, 2)
|
||||
batch_size: Any = WarmupIntRange(
|
||||
1, min(_max_sampler_batch_size(vllm_config), _SPLIT_MAX_BATCH) + 1
|
||||
)
|
||||
round: Any = WarmupIntRange(0, _SPLIT_ROUNDS)
|
||||
has_k: Any = WarmupChoices(False, True)
|
||||
logits = TritonWarmupTensor(
|
||||
torch.float32,
|
||||
shape=(batch_size, vocab_size),
|
||||
strides=(logits_stride, 1),
|
||||
)
|
||||
return dict(
|
||||
logits=logits,
|
||||
stats=TritonWarmupTensor(torch.float32),
|
||||
parts=TritonWarmupTensor(torch.float32),
|
||||
k=TritonWarmupTensor(torch.int32) if has_k else None,
|
||||
p=TritonWarmupTensor(torch.float32),
|
||||
round=round,
|
||||
num_sm=num_sm,
|
||||
)
|
||||
|
||||
|
||||
@triton_kernel_dispatcher_with_warmup(
|
||||
kernel=_topp_sb_step_kernel,
|
||||
warmup_inputs=_topp_split_step_warmup_inputs,
|
||||
)
|
||||
def _topp_split_step(
|
||||
logits: torch.Tensor,
|
||||
stats: torch.Tensor,
|
||||
parts: torch.Tensor,
|
||||
k: torch.Tensor | None,
|
||||
p: torch.Tensor,
|
||||
round: int,
|
||||
num_sm: int,
|
||||
) -> DispatchSpec:
|
||||
batch_size, vocab_size = logits.shape
|
||||
splits = _topp_split_count(batch_size, num_sm)
|
||||
return (logits.shape[0] * splits,), dict(
|
||||
K=k if k is not None else logits,
|
||||
HAS_K=k is not None,
|
||||
S=splits,
|
||||
F=_SPLIT_FANOUT,
|
||||
NUM_ROUNDS=_SPLIT_ROUNDS,
|
||||
VOCAB_SIZE=vocab_size,
|
||||
BLOCK=2048,
|
||||
num_warps=8,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _topp_sb_mask_kernel(
|
||||
LOGITS,
|
||||
@@ -1313,6 +1514,57 @@ def _topp_sb_mask_kernel(
|
||||
tl.store(ROW + i + offs, tl.where(keep, x, MASK_VALUE), mask=mask_n)
|
||||
|
||||
|
||||
def _topp_split_mask_warmup_inputs(vllm_config: Any) -> dict[str, Any]:
|
||||
vocab_size = vllm_config.model_config.get_vocab_size()
|
||||
num_sm = num_compute_units()
|
||||
logits_stride: Any = WarmupChoices(16, 2)
|
||||
batch_size: Any = WarmupIntRange(
|
||||
1, min(_max_sampler_batch_size(vllm_config), _SPLIT_MAX_BATCH) + 1
|
||||
)
|
||||
has_k: Any = WarmupChoices(False, True)
|
||||
logits = TritonWarmupTensor(
|
||||
torch.float32,
|
||||
shape=(batch_size, vocab_size),
|
||||
strides=(logits_stride, 1),
|
||||
)
|
||||
return dict(
|
||||
logits=logits,
|
||||
stats=TritonWarmupTensor(torch.float32),
|
||||
parts=TritonWarmupTensor(torch.float32),
|
||||
k=TritonWarmupTensor(torch.int32) if has_k else None,
|
||||
p=TritonWarmupTensor(torch.float32),
|
||||
mask_value=float("-inf"),
|
||||
num_sm=num_sm,
|
||||
)
|
||||
|
||||
|
||||
@triton_kernel_dispatcher_with_warmup(
|
||||
kernel=_topp_sb_mask_kernel,
|
||||
warmup_inputs=_topp_split_mask_warmup_inputs,
|
||||
)
|
||||
def _topp_split_mask(
|
||||
logits: torch.Tensor,
|
||||
stats: torch.Tensor,
|
||||
parts: torch.Tensor,
|
||||
k: torch.Tensor | None,
|
||||
p: torch.Tensor,
|
||||
mask_value: float,
|
||||
num_sm: int,
|
||||
) -> DispatchSpec:
|
||||
batch_size, vocab_size = logits.shape
|
||||
splits = _topp_split_count(batch_size, num_sm)
|
||||
return (logits.shape[0] * splits,), dict(
|
||||
K=k if k is not None else logits,
|
||||
HAS_K=k is not None,
|
||||
S=splits,
|
||||
F=_SPLIT_FANOUT,
|
||||
NUM_ROUNDS=_SPLIT_ROUNDS,
|
||||
VOCAB_SIZE=vocab_size,
|
||||
BLOCK=8192,
|
||||
num_warps=8,
|
||||
)
|
||||
|
||||
|
||||
def _apply_topp_split(
|
||||
logits: torch.Tensor,
|
||||
k: torch.Tensor | None,
|
||||
@@ -1321,11 +1573,6 @@ def _apply_topp_split(
|
||||
num_sm: int,
|
||||
) -> None:
|
||||
"""Split-row top-p pipeline for small batches; masks logits in place."""
|
||||
batch_size, vocab_size = logits.shape
|
||||
splits = 1
|
||||
while splits < _SPLIT_MAX_SPLITS and splits * 2 * batch_size <= num_sm:
|
||||
splits *= 2
|
||||
|
||||
ws = _TRITON_SPLIT_CACHE.get(logits.device)
|
||||
if ws is None:
|
||||
ws = {
|
||||
@@ -1336,53 +1583,31 @@ def _apply_topp_split(
|
||||
}
|
||||
_TRITON_SPLIT_CACHE[logits.device] = ws
|
||||
|
||||
k_ptr = k if k is not None else logits # dummy pointer when HAS_K=False
|
||||
has_k = k is not None
|
||||
|
||||
_topp_sb_stats_kernel[(batch_size * splits,)](
|
||||
_topp_split_stats(
|
||||
logits,
|
||||
logits.stride(0),
|
||||
ws["stats"],
|
||||
k_ptr,
|
||||
k,
|
||||
p,
|
||||
HAS_K=has_k,
|
||||
VOCAB_SIZE=vocab_size,
|
||||
S=splits,
|
||||
BLOCK=8192,
|
||||
num_warps=8,
|
||||
num_sm,
|
||||
)
|
||||
for round_i in range(_SPLIT_ROUNDS):
|
||||
_topp_sb_step_kernel[(batch_size * splits,)](
|
||||
_topp_split_step(
|
||||
logits,
|
||||
logits.stride(0),
|
||||
ws["stats"],
|
||||
ws["parts"],
|
||||
k_ptr,
|
||||
k,
|
||||
p,
|
||||
round_i,
|
||||
HAS_K=has_k,
|
||||
S=splits,
|
||||
F=_SPLIT_FANOUT,
|
||||
NUM_ROUNDS=_SPLIT_ROUNDS,
|
||||
VOCAB_SIZE=vocab_size,
|
||||
BLOCK=2048,
|
||||
num_warps=8,
|
||||
num_sm,
|
||||
)
|
||||
_topp_sb_mask_kernel[(batch_size * splits,)](
|
||||
_topp_split_mask(
|
||||
logits,
|
||||
logits.stride(0),
|
||||
ws["stats"],
|
||||
ws["parts"],
|
||||
k_ptr,
|
||||
k,
|
||||
p,
|
||||
HAS_K=has_k,
|
||||
MASK_VALUE=mask_value,
|
||||
S=splits,
|
||||
F=_SPLIT_FANOUT,
|
||||
NUM_ROUNDS=_SPLIT_ROUNDS,
|
||||
VOCAB_SIZE=vocab_size,
|
||||
BLOCK=8192,
|
||||
num_warps=8,
|
||||
mask_value,
|
||||
num_sm,
|
||||
)
|
||||
|
||||
|
||||
@@ -1477,39 +1702,15 @@ def apply_top_k_top_p_triton(
|
||||
else:
|
||||
normal_cdf_to_sigma_table, percentile_to_std_table = tables
|
||||
|
||||
# Smaller tiles compile and run faster on CPU; GPU benefits from larger tiles.
|
||||
# On XPU, large BLOCK_SIZE causes precision loss in the single-pass pivot
|
||||
# approximation; use smaller tiles for accurate top-p results.
|
||||
launch_kwargs = {}
|
||||
if logits.device.type == "cpu":
|
||||
block_size, block_size_trunc = 256, 128
|
||||
elif logits.device.type == "xpu":
|
||||
block_size, block_size_trunc = 4096, 2048
|
||||
else:
|
||||
block_size, block_size_trunc = 8192, 4096
|
||||
# Each program serially sweeps the vocab row in BLOCK_SIZE tiles, so
|
||||
# per-tile latency bounds kernel latency, and Triton's default of 4
|
||||
# warps leaves an 8192-wide tile at 16 elements per lane. 8 warps is
|
||||
# faster on every arch measured (SM90, SM100, SM120, gfx950); 16 is not.
|
||||
launch_kwargs["num_warps"] = 8
|
||||
|
||||
_topk_topp_kernel[(NUM_PROGRAMS,)](
|
||||
_topk_topp(
|
||||
logits,
|
||||
logits.stride(0),
|
||||
buffer,
|
||||
percentile_to_std_table,
|
||||
normal_cdf_to_sigma_table,
|
||||
k_ptr,
|
||||
p_ptr,
|
||||
BATCH_SIZE=batch_size,
|
||||
MASK_VALUE=mask_value,
|
||||
VOCAB_SIZE=vocab_size,
|
||||
BLOCK_SIZE=block_size,
|
||||
BLOCK_SIZE_TRUNC=block_size_trunc,
|
||||
TOPK_ENABLED=topk_enabled,
|
||||
TOPP_ENABLED=topp_enabled,
|
||||
SPLIT_COVERS_PONLY=use_split,
|
||||
**launch_kwargs,
|
||||
k_ptr if topk_enabled else None,
|
||||
p_ptr if topp_enabled else None,
|
||||
mask_value,
|
||||
num_sm,
|
||||
)
|
||||
if use_split:
|
||||
_apply_topp_split(logits, k_ptr, p_ptr, mask_value, num_sm)
|
||||
|
||||
@@ -5,13 +5,19 @@ from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config.model import PROCESSED_LOGPROBS_MODES
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.warmup.jit_warmup import WarmupChoices
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
DispatchSpec,
|
||||
TritonWarmupTensor,
|
||||
triton_kernel_dispatcher_with_warmup,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.outputs import LogprobsLists, LogprobsTensors, SamplerOutput
|
||||
from vllm.v1.sample.logits_processor.builtin import MinTokensLogitsProcessor
|
||||
@@ -88,6 +94,14 @@ class RejectionSampler(nn.Module):
|
||||
device=device,
|
||||
)
|
||||
self.synthetic_mode = self.synthetic_conditional_rates is not None
|
||||
_rejection_greedy_sample.register_warmup(synthetic_mode=self.synthetic_mode)
|
||||
_rejection_random_sample.register_warmup(
|
||||
synthetic_mode=self.synthetic_mode,
|
||||
)
|
||||
_expand.register_warmup()
|
||||
_sample_recovered_tokens.register_warmup(
|
||||
use_fp64_gumbel=self.use_fp64_gumbel,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -453,7 +467,7 @@ def rejection_sample(
|
||||
if not sampling_metadata.all_random:
|
||||
# Rejection sampling for greedy sampling requests.
|
||||
target_argmax = target_logits.argmax(dim=-1)
|
||||
rejection_greedy_sample_kernel[(batch_size,)](
|
||||
_rejection_greedy_sample(
|
||||
output_token_ids,
|
||||
cu_num_draft_tokens,
|
||||
draft_token_ids,
|
||||
@@ -463,7 +477,6 @@ def rejection_sample(
|
||||
max_spec_len,
|
||||
uniform_probs,
|
||||
synthetic_conditional_rates,
|
||||
SYNTHETIC_MODE=synthetic_mode,
|
||||
)
|
||||
if sampling_metadata.all_greedy:
|
||||
return output_token_ids
|
||||
@@ -488,7 +501,7 @@ def rejection_sample(
|
||||
|
||||
# Rejection sampling for random sampling requests.
|
||||
assert uniform_probs is not None
|
||||
rejection_random_sample_kernel[(batch_size,)](
|
||||
_rejection_random_sample(
|
||||
output_token_ids,
|
||||
cu_num_draft_tokens,
|
||||
draft_token_ids,
|
||||
@@ -499,10 +512,7 @@ def rejection_sample(
|
||||
uniform_probs,
|
||||
is_greedy,
|
||||
max_spec_len,
|
||||
vocab_size,
|
||||
synthetic_conditional_rates,
|
||||
NO_DRAFT_PROBS=draft_probs is None,
|
||||
SYNTHETIC_MODE=synthetic_mode,
|
||||
)
|
||||
return output_token_ids
|
||||
|
||||
@@ -594,13 +604,12 @@ def expand_batch_to_tokens(
|
||||
batch_size = x.shape[0]
|
||||
assert cu_num_tokens.shape[0] == batch_size
|
||||
expanded_x = x.new_empty(num_tokens)
|
||||
expand_kernel[(batch_size,)](
|
||||
_expand(
|
||||
expanded_x,
|
||||
x,
|
||||
cu_num_tokens,
|
||||
replace_from,
|
||||
replace_to,
|
||||
MAX_NUM_TOKENS=MAX_SPEC_LEN, # To avoid recompilation.
|
||||
)
|
||||
return expanded_x
|
||||
|
||||
@@ -694,18 +703,14 @@ def sample_recovered_tokens(
|
||||
inv_q = q.reciprocal()
|
||||
|
||||
recovered_token_ids = torch.empty_like(draft_token_ids)
|
||||
BLOCK_SIZE = 8192
|
||||
sample_recovered_tokens_kernel[(batch_size, max_spec_len)](
|
||||
_sample_recovered_tokens(
|
||||
recovered_token_ids,
|
||||
cu_num_draft_tokens,
|
||||
draft_token_ids,
|
||||
draft_probs,
|
||||
target_probs,
|
||||
inv_q,
|
||||
vocab_size,
|
||||
BLOCK_SIZE,
|
||||
NO_DRAFT_PROBS=draft_probs is None,
|
||||
USE_FP64_GUMBEL=use_fp64_gumbel,
|
||||
max_spec_len=max_spec_len,
|
||||
)
|
||||
return recovered_token_ids
|
||||
|
||||
@@ -769,6 +774,50 @@ def rejection_greedy_sample_kernel(
|
||||
)
|
||||
|
||||
|
||||
def _rejection_greedy_sample_warmup_inputs(
|
||||
*, synthetic_mode: bool
|
||||
) -> dict[str, object]:
|
||||
all_greedy = WarmupChoices(False, True)
|
||||
int32 = TritonWarmupTensor(torch.int32)
|
||||
return dict(
|
||||
output_token_ids=int32,
|
||||
cu_num_draft_tokens=int32,
|
||||
draft_token_ids=int32,
|
||||
target_argmax=TritonWarmupTensor(torch.int64),
|
||||
bonus_token_ids=TritonWarmupTensor(torch.int64),
|
||||
is_greedy=None if all_greedy else TritonWarmupTensor(torch.bool),
|
||||
max_spec_len=5,
|
||||
uniform_probs=(
|
||||
None
|
||||
if all_greedy and not synthetic_mode
|
||||
else TritonWarmupTensor(torch.float64)
|
||||
),
|
||||
synthetic_conditional_rates=(
|
||||
TritonWarmupTensor(torch.float32) if synthetic_mode else None
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@triton_kernel_dispatcher_with_warmup(
|
||||
kernel=rejection_greedy_sample_kernel,
|
||||
warmup_inputs=_rejection_greedy_sample_warmup_inputs,
|
||||
)
|
||||
def _rejection_greedy_sample(
|
||||
output_token_ids: torch.Tensor,
|
||||
cu_num_draft_tokens: torch.Tensor,
|
||||
draft_token_ids: torch.Tensor,
|
||||
target_argmax: torch.Tensor,
|
||||
bonus_token_ids: torch.Tensor,
|
||||
is_greedy: torch.Tensor | None,
|
||||
max_spec_len: int,
|
||||
uniform_probs: torch.Tensor | None,
|
||||
synthetic_conditional_rates: torch.Tensor | None,
|
||||
) -> DispatchSpec:
|
||||
return (output_token_ids.shape[0],), dict(
|
||||
SYNTHETIC_MODE=synthetic_conditional_rates is not None,
|
||||
)
|
||||
|
||||
|
||||
# NOTE(woosuk): Avoid specialization to prevent unnecessary recompilation.
|
||||
@triton.jit(do_not_specialize=["max_spec_len"])
|
||||
def rejection_random_sample_kernel(
|
||||
@@ -845,6 +894,57 @@ def rejection_random_sample_kernel(
|
||||
)
|
||||
|
||||
|
||||
def _rejection_random_sample_warmup_inputs(
|
||||
*, synthetic_mode: bool
|
||||
) -> dict[str, object]:
|
||||
vocab_size: Any = WarmupChoices(2, 16)
|
||||
no_draft_probs = WarmupChoices(False, True)
|
||||
int32 = TritonWarmupTensor(torch.int32)
|
||||
probabilities = TritonWarmupTensor(
|
||||
torch.float32,
|
||||
shape=(1, vocab_size),
|
||||
)
|
||||
return dict(
|
||||
output_token_ids=int32,
|
||||
cu_num_draft_tokens=int32,
|
||||
draft_token_ids=int32,
|
||||
draft_probs=None if no_draft_probs else probabilities,
|
||||
target_probs=probabilities,
|
||||
bonus_token_ids=TritonWarmupTensor(torch.int64),
|
||||
recovered_token_ids=int32,
|
||||
uniform_probs=TritonWarmupTensor(torch.float64),
|
||||
is_greedy=TritonWarmupTensor(torch.bool),
|
||||
max_spec_len=5,
|
||||
synthetic_conditional_rates=(
|
||||
TritonWarmupTensor(torch.float32) if synthetic_mode else None
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@triton_kernel_dispatcher_with_warmup(
|
||||
kernel=rejection_random_sample_kernel,
|
||||
warmup_inputs=_rejection_random_sample_warmup_inputs,
|
||||
)
|
||||
def _rejection_random_sample(
|
||||
output_token_ids: torch.Tensor,
|
||||
cu_num_draft_tokens: torch.Tensor,
|
||||
draft_token_ids: torch.Tensor,
|
||||
draft_probs: torch.Tensor | None,
|
||||
target_probs: torch.Tensor,
|
||||
bonus_token_ids: torch.Tensor,
|
||||
recovered_token_ids: torch.Tensor,
|
||||
uniform_probs: torch.Tensor,
|
||||
is_greedy: torch.Tensor,
|
||||
max_spec_len: int,
|
||||
synthetic_conditional_rates: torch.Tensor | None,
|
||||
) -> DispatchSpec:
|
||||
return (output_token_ids.shape[0],), dict(
|
||||
vocab_size=target_probs.shape[1],
|
||||
NO_DRAFT_PROBS=draft_probs is None,
|
||||
SYNTHETIC_MODE=synthetic_conditional_rates is not None,
|
||||
)
|
||||
|
||||
|
||||
# NOTE(woosuk): Avoid specialization to prevent unnecessary recompilation.
|
||||
@triton.jit(do_not_specialize=["replace_from", "replace_to"])
|
||||
def expand_kernel(
|
||||
@@ -869,6 +969,31 @@ def expand_kernel(
|
||||
tl.store(output_ptr + start_idx + offset, src_val, mask=offset < num_tokens)
|
||||
|
||||
|
||||
def _expand_warmup_inputs() -> dict[str, object]:
|
||||
dtype = WarmupChoices(torch.float32, torch.int32, torch.int64)
|
||||
data = TritonWarmupTensor(dtype)
|
||||
return dict(
|
||||
output=data,
|
||||
input=data,
|
||||
cu_num_tokens=TritonWarmupTensor(torch.int32),
|
||||
replace_from=0,
|
||||
replace_to=0,
|
||||
)
|
||||
|
||||
|
||||
@triton_kernel_dispatcher_with_warmup(
|
||||
kernel=expand_kernel, warmup_inputs=_expand_warmup_inputs
|
||||
)
|
||||
def _expand(
|
||||
output: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
cu_num_tokens: torch.Tensor,
|
||||
replace_from: int,
|
||||
replace_to: int,
|
||||
) -> DispatchSpec:
|
||||
return (input.shape[0],), dict(MAX_NUM_TOKENS=MAX_SPEC_LEN)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def sample_recovered_tokens_kernel(
|
||||
output_token_ids_ptr, # [num_tokens]
|
||||
@@ -951,3 +1076,49 @@ def sample_recovered_tokens_kernel(
|
||||
|
||||
recovered_id = tl.minimum(recovered_id, vocab_size - 1)
|
||||
tl.store(output_token_ids_ptr + token_idx, recovered_id)
|
||||
|
||||
|
||||
def _sample_recovered_tokens_warmup_inputs(
|
||||
*, use_fp64_gumbel: bool
|
||||
) -> dict[str, object]:
|
||||
vocab_size: Any = WarmupChoices(2, 16)
|
||||
no_draft_probs = WarmupChoices(False, True)
|
||||
int32 = TritonWarmupTensor(torch.int32)
|
||||
probabilities = TritonWarmupTensor(
|
||||
torch.float32,
|
||||
shape=(1, vocab_size),
|
||||
)
|
||||
return dict(
|
||||
output_token_ids=int32,
|
||||
cu_num_draft_tokens=int32,
|
||||
draft_token_ids=int32,
|
||||
draft_probs=None if no_draft_probs else probabilities,
|
||||
target_probs=probabilities,
|
||||
inv_q=TritonWarmupTensor(
|
||||
torch.float64 if use_fp64_gumbel else torch.float32,
|
||||
shape=(1, vocab_size),
|
||||
),
|
||||
max_spec_len=5,
|
||||
)
|
||||
|
||||
|
||||
@triton_kernel_dispatcher_with_warmup(
|
||||
kernel=sample_recovered_tokens_kernel,
|
||||
warmup_inputs=_sample_recovered_tokens_warmup_inputs,
|
||||
)
|
||||
def _sample_recovered_tokens(
|
||||
output_token_ids: torch.Tensor,
|
||||
cu_num_draft_tokens: torch.Tensor,
|
||||
draft_token_ids: torch.Tensor,
|
||||
draft_probs: torch.Tensor | None,
|
||||
target_probs: torch.Tensor,
|
||||
inv_q: torch.Tensor,
|
||||
*,
|
||||
max_spec_len: int,
|
||||
) -> DispatchSpec:
|
||||
return (cu_num_draft_tokens.shape[0], max_spec_len), dict(
|
||||
vocab_size=target_probs.shape[1],
|
||||
BLOCK_SIZE=8192,
|
||||
NO_DRAFT_PROBS=draft_probs is None,
|
||||
USE_FP64_GUMBEL=inv_q.dtype == torch.float64,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user