[6/N][warmup][DSv4] Migrate sampling, and DFlash JIT kernels (#56323)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: LopezCastroRoberto <rocastro@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Roberto L. Castro
2026-09-12 17:24:24 -07:00
committed by GitHub
co-authored by Lucas Wilkinson OpenAI Codex
parent e19a3e172e
commit aed894c190
27 changed files with 2288 additions and 558 deletions
+186 -15
View File
@@ -5,13 +5,19 @@ from __future__ import annotations
from collections.abc import Sequence
from dataclasses import replace
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
import torch
import torch.nn as nn
from vllm.config.model import PROCESSED_LOGPROBS_MODES
from vllm.logger import init_logger
from vllm.model_executor.warmup.jit_warmup import WarmupChoices
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
DispatchSpec,
TritonWarmupTensor,
triton_kernel_dispatcher_with_warmup,
)
from vllm.triton_utils import tl, triton
from vllm.v1.outputs import LogprobsLists, LogprobsTensors, SamplerOutput
from vllm.v1.sample.logits_processor.builtin import MinTokensLogitsProcessor
@@ -88,6 +94,14 @@ class RejectionSampler(nn.Module):
device=device,
)
self.synthetic_mode = self.synthetic_conditional_rates is not None
_rejection_greedy_sample.register_warmup(synthetic_mode=self.synthetic_mode)
_rejection_random_sample.register_warmup(
synthetic_mode=self.synthetic_mode,
)
_expand.register_warmup()
_sample_recovered_tokens.register_warmup(
use_fp64_gumbel=self.use_fp64_gumbel,
)
def forward(
self,
@@ -453,7 +467,7 @@ def rejection_sample(
if not sampling_metadata.all_random:
# Rejection sampling for greedy sampling requests.
target_argmax = target_logits.argmax(dim=-1)
rejection_greedy_sample_kernel[(batch_size,)](
_rejection_greedy_sample(
output_token_ids,
cu_num_draft_tokens,
draft_token_ids,
@@ -463,7 +477,6 @@ def rejection_sample(
max_spec_len,
uniform_probs,
synthetic_conditional_rates,
SYNTHETIC_MODE=synthetic_mode,
)
if sampling_metadata.all_greedy:
return output_token_ids
@@ -488,7 +501,7 @@ def rejection_sample(
# Rejection sampling for random sampling requests.
assert uniform_probs is not None
rejection_random_sample_kernel[(batch_size,)](
_rejection_random_sample(
output_token_ids,
cu_num_draft_tokens,
draft_token_ids,
@@ -499,10 +512,7 @@ def rejection_sample(
uniform_probs,
is_greedy,
max_spec_len,
vocab_size,
synthetic_conditional_rates,
NO_DRAFT_PROBS=draft_probs is None,
SYNTHETIC_MODE=synthetic_mode,
)
return output_token_ids
@@ -594,13 +604,12 @@ def expand_batch_to_tokens(
batch_size = x.shape[0]
assert cu_num_tokens.shape[0] == batch_size
expanded_x = x.new_empty(num_tokens)
expand_kernel[(batch_size,)](
_expand(
expanded_x,
x,
cu_num_tokens,
replace_from,
replace_to,
MAX_NUM_TOKENS=MAX_SPEC_LEN, # To avoid recompilation.
)
return expanded_x
@@ -694,18 +703,14 @@ def sample_recovered_tokens(
inv_q = q.reciprocal()
recovered_token_ids = torch.empty_like(draft_token_ids)
BLOCK_SIZE = 8192
sample_recovered_tokens_kernel[(batch_size, max_spec_len)](
_sample_recovered_tokens(
recovered_token_ids,
cu_num_draft_tokens,
draft_token_ids,
draft_probs,
target_probs,
inv_q,
vocab_size,
BLOCK_SIZE,
NO_DRAFT_PROBS=draft_probs is None,
USE_FP64_GUMBEL=use_fp64_gumbel,
max_spec_len=max_spec_len,
)
return recovered_token_ids
@@ -769,6 +774,50 @@ def rejection_greedy_sample_kernel(
)
def _rejection_greedy_sample_warmup_inputs(
*, synthetic_mode: bool
) -> dict[str, object]:
all_greedy = WarmupChoices(False, True)
int32 = TritonWarmupTensor(torch.int32)
return dict(
output_token_ids=int32,
cu_num_draft_tokens=int32,
draft_token_ids=int32,
target_argmax=TritonWarmupTensor(torch.int64),
bonus_token_ids=TritonWarmupTensor(torch.int64),
is_greedy=None if all_greedy else TritonWarmupTensor(torch.bool),
max_spec_len=5,
uniform_probs=(
None
if all_greedy and not synthetic_mode
else TritonWarmupTensor(torch.float64)
),
synthetic_conditional_rates=(
TritonWarmupTensor(torch.float32) if synthetic_mode else None
),
)
@triton_kernel_dispatcher_with_warmup(
kernel=rejection_greedy_sample_kernel,
warmup_inputs=_rejection_greedy_sample_warmup_inputs,
)
def _rejection_greedy_sample(
output_token_ids: torch.Tensor,
cu_num_draft_tokens: torch.Tensor,
draft_token_ids: torch.Tensor,
target_argmax: torch.Tensor,
bonus_token_ids: torch.Tensor,
is_greedy: torch.Tensor | None,
max_spec_len: int,
uniform_probs: torch.Tensor | None,
synthetic_conditional_rates: torch.Tensor | None,
) -> DispatchSpec:
return (output_token_ids.shape[0],), dict(
SYNTHETIC_MODE=synthetic_conditional_rates is not None,
)
# NOTE(woosuk): Avoid specialization to prevent unnecessary recompilation.
@triton.jit(do_not_specialize=["max_spec_len"])
def rejection_random_sample_kernel(
@@ -845,6 +894,57 @@ def rejection_random_sample_kernel(
)
def _rejection_random_sample_warmup_inputs(
*, synthetic_mode: bool
) -> dict[str, object]:
vocab_size: Any = WarmupChoices(2, 16)
no_draft_probs = WarmupChoices(False, True)
int32 = TritonWarmupTensor(torch.int32)
probabilities = TritonWarmupTensor(
torch.float32,
shape=(1, vocab_size),
)
return dict(
output_token_ids=int32,
cu_num_draft_tokens=int32,
draft_token_ids=int32,
draft_probs=None if no_draft_probs else probabilities,
target_probs=probabilities,
bonus_token_ids=TritonWarmupTensor(torch.int64),
recovered_token_ids=int32,
uniform_probs=TritonWarmupTensor(torch.float64),
is_greedy=TritonWarmupTensor(torch.bool),
max_spec_len=5,
synthetic_conditional_rates=(
TritonWarmupTensor(torch.float32) if synthetic_mode else None
),
)
@triton_kernel_dispatcher_with_warmup(
kernel=rejection_random_sample_kernel,
warmup_inputs=_rejection_random_sample_warmup_inputs,
)
def _rejection_random_sample(
output_token_ids: torch.Tensor,
cu_num_draft_tokens: torch.Tensor,
draft_token_ids: torch.Tensor,
draft_probs: torch.Tensor | None,
target_probs: torch.Tensor,
bonus_token_ids: torch.Tensor,
recovered_token_ids: torch.Tensor,
uniform_probs: torch.Tensor,
is_greedy: torch.Tensor,
max_spec_len: int,
synthetic_conditional_rates: torch.Tensor | None,
) -> DispatchSpec:
return (output_token_ids.shape[0],), dict(
vocab_size=target_probs.shape[1],
NO_DRAFT_PROBS=draft_probs is None,
SYNTHETIC_MODE=synthetic_conditional_rates is not None,
)
# NOTE(woosuk): Avoid specialization to prevent unnecessary recompilation.
@triton.jit(do_not_specialize=["replace_from", "replace_to"])
def expand_kernel(
@@ -869,6 +969,31 @@ def expand_kernel(
tl.store(output_ptr + start_idx + offset, src_val, mask=offset < num_tokens)
def _expand_warmup_inputs() -> dict[str, object]:
dtype = WarmupChoices(torch.float32, torch.int32, torch.int64)
data = TritonWarmupTensor(dtype)
return dict(
output=data,
input=data,
cu_num_tokens=TritonWarmupTensor(torch.int32),
replace_from=0,
replace_to=0,
)
@triton_kernel_dispatcher_with_warmup(
kernel=expand_kernel, warmup_inputs=_expand_warmup_inputs
)
def _expand(
output: torch.Tensor,
input: torch.Tensor,
cu_num_tokens: torch.Tensor,
replace_from: int,
replace_to: int,
) -> DispatchSpec:
return (input.shape[0],), dict(MAX_NUM_TOKENS=MAX_SPEC_LEN)
@triton.jit
def sample_recovered_tokens_kernel(
output_token_ids_ptr, # [num_tokens]
@@ -951,3 +1076,49 @@ def sample_recovered_tokens_kernel(
recovered_id = tl.minimum(recovered_id, vocab_size - 1)
tl.store(output_token_ids_ptr + token_idx, recovered_id)
def _sample_recovered_tokens_warmup_inputs(
*, use_fp64_gumbel: bool
) -> dict[str, object]:
vocab_size: Any = WarmupChoices(2, 16)
no_draft_probs = WarmupChoices(False, True)
int32 = TritonWarmupTensor(torch.int32)
probabilities = TritonWarmupTensor(
torch.float32,
shape=(1, vocab_size),
)
return dict(
output_token_ids=int32,
cu_num_draft_tokens=int32,
draft_token_ids=int32,
draft_probs=None if no_draft_probs else probabilities,
target_probs=probabilities,
inv_q=TritonWarmupTensor(
torch.float64 if use_fp64_gumbel else torch.float32,
shape=(1, vocab_size),
),
max_spec_len=5,
)
@triton_kernel_dispatcher_with_warmup(
kernel=sample_recovered_tokens_kernel,
warmup_inputs=_sample_recovered_tokens_warmup_inputs,
)
def _sample_recovered_tokens(
output_token_ids: torch.Tensor,
cu_num_draft_tokens: torch.Tensor,
draft_token_ids: torch.Tensor,
draft_probs: torch.Tensor | None,
target_probs: torch.Tensor,
inv_q: torch.Tensor,
*,
max_spec_len: int,
) -> DispatchSpec:
return (cu_num_draft_tokens.shape[0], max_spec_len), dict(
vocab_size=target_probs.shape[1],
BLOCK_SIZE=8192,
NO_DRAFT_PROBS=draft_probs is None,
USE_FP64_GUMBEL=inv_q.dtype == torch.float64,
)