[6/N][warmup][DSv4] Migrate sampling, and DFlash JIT kernels (#56323)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: LopezCastroRoberto <rocastro@redhat.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Lucas Wilkinson
OpenAI Codex
parent
e19a3e172e
commit
aed894c190
@@ -5,13 +5,19 @@ from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config.model import PROCESSED_LOGPROBS_MODES
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.warmup.jit_warmup import WarmupChoices
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
DispatchSpec,
|
||||
TritonWarmupTensor,
|
||||
triton_kernel_dispatcher_with_warmup,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.outputs import LogprobsLists, LogprobsTensors, SamplerOutput
|
||||
from vllm.v1.sample.logits_processor.builtin import MinTokensLogitsProcessor
|
||||
@@ -88,6 +94,14 @@ class RejectionSampler(nn.Module):
|
||||
device=device,
|
||||
)
|
||||
self.synthetic_mode = self.synthetic_conditional_rates is not None
|
||||
_rejection_greedy_sample.register_warmup(synthetic_mode=self.synthetic_mode)
|
||||
_rejection_random_sample.register_warmup(
|
||||
synthetic_mode=self.synthetic_mode,
|
||||
)
|
||||
_expand.register_warmup()
|
||||
_sample_recovered_tokens.register_warmup(
|
||||
use_fp64_gumbel=self.use_fp64_gumbel,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -453,7 +467,7 @@ def rejection_sample(
|
||||
if not sampling_metadata.all_random:
|
||||
# Rejection sampling for greedy sampling requests.
|
||||
target_argmax = target_logits.argmax(dim=-1)
|
||||
rejection_greedy_sample_kernel[(batch_size,)](
|
||||
_rejection_greedy_sample(
|
||||
output_token_ids,
|
||||
cu_num_draft_tokens,
|
||||
draft_token_ids,
|
||||
@@ -463,7 +477,6 @@ def rejection_sample(
|
||||
max_spec_len,
|
||||
uniform_probs,
|
||||
synthetic_conditional_rates,
|
||||
SYNTHETIC_MODE=synthetic_mode,
|
||||
)
|
||||
if sampling_metadata.all_greedy:
|
||||
return output_token_ids
|
||||
@@ -488,7 +501,7 @@ def rejection_sample(
|
||||
|
||||
# Rejection sampling for random sampling requests.
|
||||
assert uniform_probs is not None
|
||||
rejection_random_sample_kernel[(batch_size,)](
|
||||
_rejection_random_sample(
|
||||
output_token_ids,
|
||||
cu_num_draft_tokens,
|
||||
draft_token_ids,
|
||||
@@ -499,10 +512,7 @@ def rejection_sample(
|
||||
uniform_probs,
|
||||
is_greedy,
|
||||
max_spec_len,
|
||||
vocab_size,
|
||||
synthetic_conditional_rates,
|
||||
NO_DRAFT_PROBS=draft_probs is None,
|
||||
SYNTHETIC_MODE=synthetic_mode,
|
||||
)
|
||||
return output_token_ids
|
||||
|
||||
@@ -594,13 +604,12 @@ def expand_batch_to_tokens(
|
||||
batch_size = x.shape[0]
|
||||
assert cu_num_tokens.shape[0] == batch_size
|
||||
expanded_x = x.new_empty(num_tokens)
|
||||
expand_kernel[(batch_size,)](
|
||||
_expand(
|
||||
expanded_x,
|
||||
x,
|
||||
cu_num_tokens,
|
||||
replace_from,
|
||||
replace_to,
|
||||
MAX_NUM_TOKENS=MAX_SPEC_LEN, # To avoid recompilation.
|
||||
)
|
||||
return expanded_x
|
||||
|
||||
@@ -694,18 +703,14 @@ def sample_recovered_tokens(
|
||||
inv_q = q.reciprocal()
|
||||
|
||||
recovered_token_ids = torch.empty_like(draft_token_ids)
|
||||
BLOCK_SIZE = 8192
|
||||
sample_recovered_tokens_kernel[(batch_size, max_spec_len)](
|
||||
_sample_recovered_tokens(
|
||||
recovered_token_ids,
|
||||
cu_num_draft_tokens,
|
||||
draft_token_ids,
|
||||
draft_probs,
|
||||
target_probs,
|
||||
inv_q,
|
||||
vocab_size,
|
||||
BLOCK_SIZE,
|
||||
NO_DRAFT_PROBS=draft_probs is None,
|
||||
USE_FP64_GUMBEL=use_fp64_gumbel,
|
||||
max_spec_len=max_spec_len,
|
||||
)
|
||||
return recovered_token_ids
|
||||
|
||||
@@ -769,6 +774,50 @@ def rejection_greedy_sample_kernel(
|
||||
)
|
||||
|
||||
|
||||
def _rejection_greedy_sample_warmup_inputs(
|
||||
*, synthetic_mode: bool
|
||||
) -> dict[str, object]:
|
||||
all_greedy = WarmupChoices(False, True)
|
||||
int32 = TritonWarmupTensor(torch.int32)
|
||||
return dict(
|
||||
output_token_ids=int32,
|
||||
cu_num_draft_tokens=int32,
|
||||
draft_token_ids=int32,
|
||||
target_argmax=TritonWarmupTensor(torch.int64),
|
||||
bonus_token_ids=TritonWarmupTensor(torch.int64),
|
||||
is_greedy=None if all_greedy else TritonWarmupTensor(torch.bool),
|
||||
max_spec_len=5,
|
||||
uniform_probs=(
|
||||
None
|
||||
if all_greedy and not synthetic_mode
|
||||
else TritonWarmupTensor(torch.float64)
|
||||
),
|
||||
synthetic_conditional_rates=(
|
||||
TritonWarmupTensor(torch.float32) if synthetic_mode else None
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@triton_kernel_dispatcher_with_warmup(
|
||||
kernel=rejection_greedy_sample_kernel,
|
||||
warmup_inputs=_rejection_greedy_sample_warmup_inputs,
|
||||
)
|
||||
def _rejection_greedy_sample(
|
||||
output_token_ids: torch.Tensor,
|
||||
cu_num_draft_tokens: torch.Tensor,
|
||||
draft_token_ids: torch.Tensor,
|
||||
target_argmax: torch.Tensor,
|
||||
bonus_token_ids: torch.Tensor,
|
||||
is_greedy: torch.Tensor | None,
|
||||
max_spec_len: int,
|
||||
uniform_probs: torch.Tensor | None,
|
||||
synthetic_conditional_rates: torch.Tensor | None,
|
||||
) -> DispatchSpec:
|
||||
return (output_token_ids.shape[0],), dict(
|
||||
SYNTHETIC_MODE=synthetic_conditional_rates is not None,
|
||||
)
|
||||
|
||||
|
||||
# NOTE(woosuk): Avoid specialization to prevent unnecessary recompilation.
|
||||
@triton.jit(do_not_specialize=["max_spec_len"])
|
||||
def rejection_random_sample_kernel(
|
||||
@@ -845,6 +894,57 @@ def rejection_random_sample_kernel(
|
||||
)
|
||||
|
||||
|
||||
def _rejection_random_sample_warmup_inputs(
|
||||
*, synthetic_mode: bool
|
||||
) -> dict[str, object]:
|
||||
vocab_size: Any = WarmupChoices(2, 16)
|
||||
no_draft_probs = WarmupChoices(False, True)
|
||||
int32 = TritonWarmupTensor(torch.int32)
|
||||
probabilities = TritonWarmupTensor(
|
||||
torch.float32,
|
||||
shape=(1, vocab_size),
|
||||
)
|
||||
return dict(
|
||||
output_token_ids=int32,
|
||||
cu_num_draft_tokens=int32,
|
||||
draft_token_ids=int32,
|
||||
draft_probs=None if no_draft_probs else probabilities,
|
||||
target_probs=probabilities,
|
||||
bonus_token_ids=TritonWarmupTensor(torch.int64),
|
||||
recovered_token_ids=int32,
|
||||
uniform_probs=TritonWarmupTensor(torch.float64),
|
||||
is_greedy=TritonWarmupTensor(torch.bool),
|
||||
max_spec_len=5,
|
||||
synthetic_conditional_rates=(
|
||||
TritonWarmupTensor(torch.float32) if synthetic_mode else None
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@triton_kernel_dispatcher_with_warmup(
|
||||
kernel=rejection_random_sample_kernel,
|
||||
warmup_inputs=_rejection_random_sample_warmup_inputs,
|
||||
)
|
||||
def _rejection_random_sample(
|
||||
output_token_ids: torch.Tensor,
|
||||
cu_num_draft_tokens: torch.Tensor,
|
||||
draft_token_ids: torch.Tensor,
|
||||
draft_probs: torch.Tensor | None,
|
||||
target_probs: torch.Tensor,
|
||||
bonus_token_ids: torch.Tensor,
|
||||
recovered_token_ids: torch.Tensor,
|
||||
uniform_probs: torch.Tensor,
|
||||
is_greedy: torch.Tensor,
|
||||
max_spec_len: int,
|
||||
synthetic_conditional_rates: torch.Tensor | None,
|
||||
) -> DispatchSpec:
|
||||
return (output_token_ids.shape[0],), dict(
|
||||
vocab_size=target_probs.shape[1],
|
||||
NO_DRAFT_PROBS=draft_probs is None,
|
||||
SYNTHETIC_MODE=synthetic_conditional_rates is not None,
|
||||
)
|
||||
|
||||
|
||||
# NOTE(woosuk): Avoid specialization to prevent unnecessary recompilation.
|
||||
@triton.jit(do_not_specialize=["replace_from", "replace_to"])
|
||||
def expand_kernel(
|
||||
@@ -869,6 +969,31 @@ def expand_kernel(
|
||||
tl.store(output_ptr + start_idx + offset, src_val, mask=offset < num_tokens)
|
||||
|
||||
|
||||
def _expand_warmup_inputs() -> dict[str, object]:
|
||||
dtype = WarmupChoices(torch.float32, torch.int32, torch.int64)
|
||||
data = TritonWarmupTensor(dtype)
|
||||
return dict(
|
||||
output=data,
|
||||
input=data,
|
||||
cu_num_tokens=TritonWarmupTensor(torch.int32),
|
||||
replace_from=0,
|
||||
replace_to=0,
|
||||
)
|
||||
|
||||
|
||||
@triton_kernel_dispatcher_with_warmup(
|
||||
kernel=expand_kernel, warmup_inputs=_expand_warmup_inputs
|
||||
)
|
||||
def _expand(
|
||||
output: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
cu_num_tokens: torch.Tensor,
|
||||
replace_from: int,
|
||||
replace_to: int,
|
||||
) -> DispatchSpec:
|
||||
return (input.shape[0],), dict(MAX_NUM_TOKENS=MAX_SPEC_LEN)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def sample_recovered_tokens_kernel(
|
||||
output_token_ids_ptr, # [num_tokens]
|
||||
@@ -951,3 +1076,49 @@ def sample_recovered_tokens_kernel(
|
||||
|
||||
recovered_id = tl.minimum(recovered_id, vocab_size - 1)
|
||||
tl.store(output_token_ids_ptr + token_idx, recovered_id)
|
||||
|
||||
|
||||
def _sample_recovered_tokens_warmup_inputs(
|
||||
*, use_fp64_gumbel: bool
|
||||
) -> dict[str, object]:
|
||||
vocab_size: Any = WarmupChoices(2, 16)
|
||||
no_draft_probs = WarmupChoices(False, True)
|
||||
int32 = TritonWarmupTensor(torch.int32)
|
||||
probabilities = TritonWarmupTensor(
|
||||
torch.float32,
|
||||
shape=(1, vocab_size),
|
||||
)
|
||||
return dict(
|
||||
output_token_ids=int32,
|
||||
cu_num_draft_tokens=int32,
|
||||
draft_token_ids=int32,
|
||||
draft_probs=None if no_draft_probs else probabilities,
|
||||
target_probs=probabilities,
|
||||
inv_q=TritonWarmupTensor(
|
||||
torch.float64 if use_fp64_gumbel else torch.float32,
|
||||
shape=(1, vocab_size),
|
||||
),
|
||||
max_spec_len=5,
|
||||
)
|
||||
|
||||
|
||||
@triton_kernel_dispatcher_with_warmup(
|
||||
kernel=sample_recovered_tokens_kernel,
|
||||
warmup_inputs=_sample_recovered_tokens_warmup_inputs,
|
||||
)
|
||||
def _sample_recovered_tokens(
|
||||
output_token_ids: torch.Tensor,
|
||||
cu_num_draft_tokens: torch.Tensor,
|
||||
draft_token_ids: torch.Tensor,
|
||||
draft_probs: torch.Tensor | None,
|
||||
target_probs: torch.Tensor,
|
||||
inv_q: torch.Tensor,
|
||||
*,
|
||||
max_spec_len: int,
|
||||
) -> DispatchSpec:
|
||||
return (cu_num_draft_tokens.shape[0], max_spec_len), dict(
|
||||
vocab_size=target_probs.shape[1],
|
||||
BLOCK_SIZE=8192,
|
||||
NO_DRAFT_PROBS=draft_probs is None,
|
||||
USE_FP64_GUMBEL=inv_q.dtype == torch.float64,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user