[6/N][warmup][DSv4] Migrate sampling, and DFlash JIT kernels (#56323)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: LopezCastroRoberto <rocastro@redhat.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Lucas Wilkinson
OpenAI Codex
parent
e19a3e172e
commit
aed894c190
@@ -1,7 +1,18 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup import (
|
||||
WarmupChoices,
|
||||
WarmupIntRange,
|
||||
)
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
DispatchSpec,
|
||||
TritonWarmupTensor,
|
||||
triton_kernel_dispatcher_with_warmup,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
@@ -85,6 +96,49 @@ def eagle_step_slot_mapping_metadata_kernel(
|
||||
tl.store(seq_lens_ptr + req_idx, new_seq_len)
|
||||
|
||||
|
||||
def _eagle_step_warmup_inputs(*, max_model_len: int, max_batch_size: int):
|
||||
block_size = WarmupIntRange(1, 257, advance=lambda value: value * 2)
|
||||
n_blocks_per_req = triton.cdiv(max_model_len, block_size)
|
||||
return dict(
|
||||
positions=TritonWarmupTensor(torch.int64),
|
||||
block_table=TritonWarmupTensor(
|
||||
torch.int32, shape=(1, n_blocks_per_req), strides=(n_blocks_per_req, 1)
|
||||
),
|
||||
block_table_stride=n_blocks_per_req,
|
||||
seq_lens=TritonWarmupTensor(torch.int32),
|
||||
out_clamped_positions=TritonWarmupTensor(torch.int64),
|
||||
out_slot_mapping=TritonWarmupTensor(torch.int64),
|
||||
block_size=block_size,
|
||||
max_model_len=max_model_len,
|
||||
n_blocks_per_req=n_blocks_per_req,
|
||||
PAD_ID=PADDING_SLOT_ID,
|
||||
batch_size=WarmupIntRange(1, max_batch_size + 1),
|
||||
input_batch_size=1,
|
||||
)
|
||||
|
||||
|
||||
@triton_kernel_dispatcher_with_warmup(
|
||||
kernel=eagle_step_slot_mapping_metadata_kernel,
|
||||
warmup_inputs=_eagle_step_warmup_inputs,
|
||||
)
|
||||
def _eagle_step_slot_mapping_metadata(
|
||||
positions: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
block_table_stride: int,
|
||||
seq_lens: torch.Tensor,
|
||||
out_clamped_positions: torch.Tensor,
|
||||
out_slot_mapping: torch.Tensor,
|
||||
*,
|
||||
block_size: int,
|
||||
max_model_len: int,
|
||||
n_blocks_per_req: int,
|
||||
PAD_ID: int,
|
||||
batch_size: int,
|
||||
input_batch_size: int,
|
||||
) -> DispatchSpec:
|
||||
return (input_batch_size,), {}
|
||||
|
||||
|
||||
def eagle_step_update_slot_mapping_and_metadata(
|
||||
positions_1d: torch.Tensor,
|
||||
block_table_tensor: torch.Tensor,
|
||||
@@ -118,7 +172,7 @@ def eagle_step_update_slot_mapping_and_metadata(
|
||||
input_batch_size = batch_size
|
||||
|
||||
n_blocks_per_req = block_table_tensor.shape[1]
|
||||
eagle_step_slot_mapping_metadata_kernel[(input_batch_size,)](
|
||||
_eagle_step_slot_mapping_metadata(
|
||||
positions_1d,
|
||||
block_table_tensor,
|
||||
block_table_tensor.stride(0),
|
||||
@@ -130,6 +184,7 @@ def eagle_step_update_slot_mapping_and_metadata(
|
||||
n_blocks_per_req=n_blocks_per_req,
|
||||
PAD_ID=PADDING_SLOT_ID,
|
||||
batch_size=batch_size,
|
||||
input_batch_size=input_batch_size,
|
||||
)
|
||||
|
||||
|
||||
@@ -175,6 +230,33 @@ def eagle_prepare_inputs_padded_kernel(
|
||||
tl.store(num_rejected_tokens_gpu_ptr + req_idx, num_rejected_tokens)
|
||||
|
||||
|
||||
def _eagle_prepare_inputs_warmup_inputs(*, max_batch_size: int):
|
||||
int32 = TritonWarmupTensor(torch.int32)
|
||||
return dict(
|
||||
cu_num_draft_tokens=int32,
|
||||
valid_sampled_tokens_count=int32,
|
||||
query_start_loc_gpu=int32,
|
||||
token_indices_to_sample=int32,
|
||||
num_rejected_tokens_gpu=int32,
|
||||
num_reqs=WarmupIntRange(1, max_batch_size + 1),
|
||||
)
|
||||
|
||||
|
||||
@triton_kernel_dispatcher_with_warmup(
|
||||
kernel=eagle_prepare_inputs_padded_kernel,
|
||||
warmup_inputs=_eagle_prepare_inputs_warmup_inputs,
|
||||
)
|
||||
def _eagle_prepare_inputs_padded(
|
||||
cu_num_draft_tokens: torch.Tensor,
|
||||
valid_sampled_tokens_count: torch.Tensor,
|
||||
query_start_loc_gpu: torch.Tensor,
|
||||
token_indices_to_sample: torch.Tensor,
|
||||
num_rejected_tokens_gpu: torch.Tensor,
|
||||
num_reqs: int,
|
||||
) -> DispatchSpec:
|
||||
return (num_reqs,), {}
|
||||
|
||||
|
||||
@triton.jit
|
||||
def eagle_prepare_next_token_padded_kernel(
|
||||
sampled_token_ids_ptr, # [num_reqs, num_sampled_tokens_per_req]
|
||||
@@ -238,6 +320,49 @@ def eagle_prepare_next_token_padded_kernel(
|
||||
tl.store(valid_sampled_tokens_count_ptr + req_idx, valid_count)
|
||||
|
||||
|
||||
def _eagle_prepare_next_token_warmup_inputs(
|
||||
*, vocab_size: int, num_sampled_tokens_per_req: int, max_batch_size: int
|
||||
):
|
||||
int32 = TritonWarmupTensor(torch.int32)
|
||||
sampled_width: Any = WarmupIntRange(1, num_sampled_tokens_per_req + 1)
|
||||
return dict(
|
||||
sampled_token_ids=TritonWarmupTensor(
|
||||
torch.int32,
|
||||
shape=(1, sampled_width),
|
||||
strides=(sampled_width, 1),
|
||||
),
|
||||
discard_request_mask=TritonWarmupTensor(torch.bool),
|
||||
backup_next_token_ids=int32,
|
||||
next_token_ids=int32,
|
||||
valid_sampled_tokens_count=int32,
|
||||
vocab_size=vocab_size,
|
||||
num_sampled_tokens_per_req=sampled_width,
|
||||
num_reqs=WarmupIntRange(1, max_batch_size + 1),
|
||||
stride_sampled_token_ids=sampled_width,
|
||||
BLOCK_SIZE_TOKENS=next_power_of_2(sampled_width),
|
||||
)
|
||||
|
||||
|
||||
@triton_kernel_dispatcher_with_warmup(
|
||||
kernel=eagle_prepare_next_token_padded_kernel,
|
||||
warmup_inputs=_eagle_prepare_next_token_warmup_inputs,
|
||||
)
|
||||
def _eagle_prepare_next_token_padded(
|
||||
sampled_token_ids: torch.Tensor,
|
||||
discard_request_mask: torch.Tensor,
|
||||
backup_next_token_ids: torch.Tensor,
|
||||
next_token_ids: torch.Tensor,
|
||||
valid_sampled_tokens_count: torch.Tensor,
|
||||
vocab_size: int,
|
||||
num_sampled_tokens_per_req: int,
|
||||
num_reqs: int,
|
||||
stride_sampled_token_ids: int,
|
||||
*,
|
||||
BLOCK_SIZE_TOKENS: int,
|
||||
) -> DispatchSpec:
|
||||
return (num_reqs,), {}
|
||||
|
||||
|
||||
def compute_new_slot_mapping(
|
||||
cad: CommonAttentionMetadata,
|
||||
new_positions: torch.Tensor,
|
||||
@@ -454,6 +579,68 @@ def copy_and_expand_eagle_inputs_kernel(
|
||||
)
|
||||
|
||||
|
||||
def _copy_and_expand_eagle_warmup_inputs(
|
||||
*,
|
||||
parallel_drafting_token_id: int,
|
||||
num_padding_slots_per_request: int,
|
||||
shift_input_ids: bool,
|
||||
max_num_tokens: int,
|
||||
):
|
||||
int32 = TritonWarmupTensor(torch.int32)
|
||||
int64 = TritonWarmupTensor(torch.int64)
|
||||
boolean = TritonWarmupTensor(torch.bool)
|
||||
return dict(
|
||||
target_token_ids_ptr=int32,
|
||||
target_positions_ptr=int64,
|
||||
next_token_ids_ptr=int32,
|
||||
out_input_ids_ptr=int32,
|
||||
out_positions_ptr=int64,
|
||||
out_is_rejected_token_mask_ptr=boolean,
|
||||
out_is_masked_token_mask_ptr=boolean,
|
||||
out_new_token_indices_ptr=int32,
|
||||
out_hidden_state_mapping_ptr=int32,
|
||||
query_start_loc_ptr=int32,
|
||||
query_end_loc_ptr=int32,
|
||||
padding_token_id=0,
|
||||
parallel_drafting_token_id=parallel_drafting_token_id,
|
||||
total_input_tokens=WarmupIntRange(1, max_num_tokens + 1),
|
||||
num_padding_slots_per_request=num_padding_slots_per_request,
|
||||
shift_input_ids=shift_input_ids,
|
||||
BLOCK_SIZE_TOKENS=WarmupIntRange(1, 257, advance=lambda value: value * 2),
|
||||
batch_size=1,
|
||||
num_blocks=1,
|
||||
)
|
||||
|
||||
|
||||
@triton_kernel_dispatcher_with_warmup(
|
||||
kernel=copy_and_expand_eagle_inputs_kernel,
|
||||
warmup_inputs=_copy_and_expand_eagle_warmup_inputs,
|
||||
)
|
||||
def _copy_and_expand_eagle_inputs(
|
||||
target_token_ids_ptr: torch.Tensor,
|
||||
target_positions_ptr: torch.Tensor,
|
||||
next_token_ids_ptr: torch.Tensor,
|
||||
out_input_ids_ptr: torch.Tensor,
|
||||
out_positions_ptr: torch.Tensor,
|
||||
out_is_rejected_token_mask_ptr: torch.Tensor,
|
||||
out_is_masked_token_mask_ptr: torch.Tensor,
|
||||
out_new_token_indices_ptr: torch.Tensor,
|
||||
out_hidden_state_mapping_ptr: torch.Tensor,
|
||||
query_start_loc_ptr: torch.Tensor,
|
||||
query_end_loc_ptr: torch.Tensor,
|
||||
padding_token_id: int,
|
||||
parallel_drafting_token_id: int,
|
||||
total_input_tokens: int,
|
||||
num_padding_slots_per_request: int,
|
||||
shift_input_ids: bool,
|
||||
*,
|
||||
BLOCK_SIZE_TOKENS: int,
|
||||
batch_size: int,
|
||||
num_blocks: int,
|
||||
) -> DispatchSpec:
|
||||
return (batch_size, num_blocks), {}
|
||||
|
||||
|
||||
@triton.jit
|
||||
def copy_and_expand_dflash_inputs_kernel(
|
||||
# Inputs
|
||||
@@ -562,6 +749,75 @@ def copy_and_expand_dflash_inputs_kernel(
|
||||
)
|
||||
|
||||
|
||||
def _copy_and_expand_dflash_warmup_inputs(
|
||||
*,
|
||||
block_table_stride: int,
|
||||
parallel_drafting_token_id: int,
|
||||
block_size: int,
|
||||
num_speculative_tokens: int,
|
||||
):
|
||||
int32 = TritonWarmupTensor(torch.int32)
|
||||
int64 = TritonWarmupTensor(torch.int64)
|
||||
triton_block_size = WarmupIntRange(1, 257, advance=lambda value: value * 2)
|
||||
has_num_rejected = WarmupChoices(False, True)
|
||||
return dict(
|
||||
next_token_ids_ptr=int32,
|
||||
target_positions_ptr=int64,
|
||||
out_input_ids_ptr=int32,
|
||||
out_context_positions_ptr=int64,
|
||||
out_query_positions_ptr=int64,
|
||||
out_context_slot_mapping_ptr=int64,
|
||||
out_query_slot_mapping_ptr=int64,
|
||||
out_token_indices_ptr=int32,
|
||||
block_table_ptr=int32,
|
||||
query_start_loc_ptr=int32,
|
||||
num_rejected_tokens_ptr=(int32 if has_num_rejected else 0),
|
||||
block_table_stride=block_table_stride,
|
||||
parallel_drafting_token_id=parallel_drafting_token_id,
|
||||
block_size=block_size,
|
||||
num_query_per_req=1 + num_speculative_tokens,
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
total_input_tokens=WarmupChoices(1, 2, 16),
|
||||
max_tokens_per_req=triton_block_size,
|
||||
triton_block_size=triton_block_size,
|
||||
has_num_rejected=has_num_rejected,
|
||||
)
|
||||
|
||||
|
||||
@triton_kernel_dispatcher_with_warmup(
|
||||
kernel=copy_and_expand_dflash_inputs_kernel,
|
||||
warmup_inputs=_copy_and_expand_dflash_warmup_inputs,
|
||||
)
|
||||
def _copy_and_expand_dflash_inputs(
|
||||
*,
|
||||
next_token_ids_ptr: torch.Tensor,
|
||||
target_positions_ptr: torch.Tensor,
|
||||
out_input_ids_ptr: torch.Tensor,
|
||||
out_context_positions_ptr: torch.Tensor,
|
||||
out_query_positions_ptr: torch.Tensor,
|
||||
out_context_slot_mapping_ptr: torch.Tensor,
|
||||
out_query_slot_mapping_ptr: torch.Tensor,
|
||||
out_token_indices_ptr: torch.Tensor,
|
||||
block_table_ptr: torch.Tensor,
|
||||
query_start_loc_ptr: torch.Tensor,
|
||||
num_rejected_tokens_ptr: torch.Tensor | int,
|
||||
block_table_stride: int,
|
||||
parallel_drafting_token_id: int,
|
||||
block_size: int,
|
||||
num_query_per_req: int,
|
||||
num_speculative_tokens: int,
|
||||
total_input_tokens: int,
|
||||
max_tokens_per_req: int,
|
||||
triton_block_size: int,
|
||||
has_num_rejected: bool,
|
||||
) -> DispatchSpec:
|
||||
num_reqs = query_start_loc_ptr.shape[0] - 1
|
||||
num_blocks = triton.cdiv(max_tokens_per_req, triton_block_size)
|
||||
return (num_reqs, num_blocks), dict(
|
||||
BLOCK_SIZE=triton_block_size,
|
||||
)
|
||||
|
||||
|
||||
@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend)
|
||||
def update_num_computed_tokens_for_batch_change(
|
||||
num_computed_tokens: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user