[6/N][warmup][DSv4] Migrate sampling, and DFlash JIT kernels (#56323)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: LopezCastroRoberto <rocastro@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Roberto L. Castro
2026-09-12 17:24:24 -07:00
committed by GitHub
co-authored by Lucas Wilkinson OpenAI Codex
parent e19a3e172e
commit aed894c190
27 changed files with 2288 additions and 558 deletions
@@ -1,8 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
from types import SimpleNamespace
from typing import Any
import numpy as np
import torch
import torch.nn as nn
@@ -10,6 +12,15 @@ from vllm.config import VllmConfig, replace
from vllm.config.compilation import CUDAGraphMode
from vllm.forward_context import BatchDescriptor, set_forward_context
from vllm.logger import init_logger
from vllm.model_executor.warmup.jit_warmup import (
WarmupChoices,
WarmupIntRange,
)
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
DispatchSpec,
TritonWarmupTensor,
triton_kernel_dispatcher_with_warmup,
)
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backend import AttentionCGSupport
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
@@ -108,6 +119,7 @@ class DFlashSpeculator(DraftModelSpeculator):
self.query_cudagraph_manager: DFlashCudaGraphManager | None = None
self.draft_kv_cache_group_id: int = -1
prepare_dflash_inputs.register_warmup(speculator=self)
@property
def attn_vllm_config(self) -> VllmConfig:
@@ -682,6 +694,64 @@ def _prepare_dflash_inputs_kernel(
tl.store(out_query_slot_mapping_ptr + block, PAD_SLOT_ID, mask=mask)
def _prepare_dflash_warmup_inputs(*, speculator: DFlashSpeculator) -> dict[str, Any]:
int32 = TritonWarmupTensor(torch.int32)
int64 = TritonWarmupTensor(torch.int64)
float32 = TritonWarmupTensor(torch.float32)
input_buffers = SimpleNamespace(
input_ids=int32, positions=int64, query_start_loc=int32, seq_lens=int32
)
gid: Any = WarmupChoices(*speculator.draft_kv_cache_group_ids)
max_target_query_len: Any = WarmupIntRange(1, 257, advance=lambda value: value * 2)
block_table_stride = int(speculator.block_tables.input_block_tables[gid].stride(0))
block_table = TritonWarmupTensor(
torch.int32,
shape=(1, block_table_stride),
strides=(block_table_stride, 1),
)
input_batch = SimpleNamespace(
num_reqs=1,
num_scheduled_tokens=np.array([max_target_query_len]),
positions=int64,
query_start_loc=int32,
idx_mapping=int64,
)
return dict(
input_buffers=input_buffers,
query_slot_mapping=int64,
context_positions=int64,
context_slot_mapping=int64,
sample_indices=int64,
sample_pos=int64,
sample_idx_mapping=int32,
temperature=float32,
seeds=int64,
input_batch=input_batch,
num_sampled=int32,
num_rejected=int32,
last_sampled=int64,
next_prefill_tokens=int32,
input_temperature=float32,
input_seeds=int64,
block_table=block_table,
block_size=speculator.block_tables.kernel_block_sizes[gid],
cp_rank=speculator.block_tables.cp_rank,
cp_size=speculator.block_tables.cp_size,
cp_interleave=speculator.block_tables.cp_interleave,
parallel_drafting_token_id=speculator.parallel_drafting_token_id,
num_query_per_req=speculator.num_query_per_req,
num_speculative_steps=speculator.num_speculative_steps,
max_num_reqs=speculator.max_num_reqs,
max_num_tokens=speculator.max_num_tokens,
max_model_len=speculator.max_model_len,
sample_from_anchor=speculator.sample_from_anchor,
)
@triton_kernel_dispatcher_with_warmup(
kernel=_prepare_dflash_inputs_kernel,
warmup_inputs=_prepare_dflash_warmup_inputs,
)
def prepare_dflash_inputs(
input_buffers: InputBuffers,
query_slot_mapping: torch.Tensor,
@@ -718,50 +788,33 @@ def prepare_dflash_inputs(
max_num_tokens: int,
max_model_len: int,
sample_from_anchor: bool = False,
) -> None:
) -> DispatchSpec:
num_reqs = input_batch.num_reqs
assert num_reqs > 0
# Cover the longest possible per-request span (ctx + query). Use the max
# per-request query length, not the total token count across the batch.
max_target_query_len = int(input_batch.num_scheduled_tokens.max())
max_tokens_per_req = max_target_query_len + num_query_per_req
BLOCK_SIZE = min(256, triton.next_power_of_2(max(1, max_tokens_per_req)))
num_blocks = triton.cdiv(max_tokens_per_req, BLOCK_SIZE)
_prepare_dflash_inputs_kernel[(num_reqs, num_blocks)](
input_buffers.input_ids,
input_buffers.positions,
input_buffers.query_start_loc,
input_buffers.seq_lens,
query_slot_mapping,
context_positions,
context_slot_mapping,
sample_indices,
sample_pos,
sample_idx_mapping,
temperature,
seeds,
input_batch.positions,
input_batch.query_start_loc,
input_batch.idx_mapping,
last_sampled,
next_prefill_tokens,
num_sampled,
num_rejected,
input_temperature,
input_seeds,
block_table,
block_table.stride(0),
parallel_drafting_token_id,
block_size,
num_query_per_req,
num_speculative_steps,
max_num_reqs,
max_num_tokens,
max_model_len,
cp_rank,
SAMPLE_FROM_ANCHOR=sample_from_anchor,
block = min(256, triton.next_power_of_2(max(1, max_tokens_per_req)))
num_blocks = triton.cdiv(max_tokens_per_req, block)
return (num_reqs, num_blocks), dict(
out_input_ids_ptr=input_buffers.input_ids,
out_query_positions_ptr=input_buffers.positions,
out_query_start_loc_ptr=input_buffers.query_start_loc,
out_seq_lens_ptr=input_buffers.seq_lens,
out_query_slot_mapping_ptr=query_slot_mapping,
out_context_positions_ptr=context_positions,
out_context_slot_mapping_ptr=context_slot_mapping,
out_sample_indices_ptr=sample_indices,
out_sample_pos_ptr=sample_pos,
out_sample_idx_mapping_ptr=sample_idx_mapping,
out_temperature_ptr=temperature,
out_seeds_ptr=seeds,
target_positions_ptr=input_batch.positions,
target_query_start_loc_ptr=input_batch.query_start_loc,
idx_mapping_ptr=input_batch.idx_mapping,
temperature_ptr=input_temperature,
seeds_ptr=input_seeds,
PAD_SLOT_ID=PAD_SLOT_ID,
CP_SIZE=cp_size,
CP_INTERLEAVE=cp_interleave,
BLOCK_SIZE=BLOCK_SIZE,
BLOCK_SIZE=block,
)
@@ -9,6 +9,11 @@ from vllm.config import VllmConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.forward_context import BatchDescriptor, set_forward_context
from vllm.logger import init_logger
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
DispatchSpec,
TritonWarmupTensor,
triton_kernel_dispatcher_with_warmup,
)
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
@@ -69,6 +74,9 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator):
self.cudagraph_manager: SpeculatorCudaGraphManager | None = None
_shift_input_ids.register_warmup()
_shift_input_embeds.register_warmup(speculator=self)
def load_draft_model(
self,
target_model: nn.Module,
@@ -1008,6 +1016,32 @@ def _shift_input_ids_kernel(
tl.store(input_ids_ptr + last_token_index, draft_token)
def _shift_input_ids_warmup_inputs() -> dict[str, Any]:
return dict(
input_ids=TritonWarmupTensor(torch.int32),
idx_mapping=TritonWarmupTensor(torch.int32),
query_start_loc=TritonWarmupTensor(torch.int32),
last_token_indices=TritonWarmupTensor(torch.int64),
draft_tokens=TritonWarmupTensor(torch.int64),
num_reqs=1,
)
@triton_kernel_dispatcher_with_warmup(
kernel=_shift_input_ids_kernel,
warmup_inputs=_shift_input_ids_warmup_inputs,
)
def _shift_input_ids(
input_ids: torch.Tensor,
idx_mapping: torch.Tensor,
query_start_loc: torch.Tensor,
last_token_indices: torch.Tensor,
draft_tokens: torch.Tensor,
num_reqs: int,
) -> DispatchSpec:
return (num_reqs,), dict(BLOCK_SIZE=1024)
@triton.jit
def _shift_input_embeds_kernel(
input_embeds_ptr,
@@ -1065,6 +1099,44 @@ def _shift_input_embeds_kernel(
)
def _shift_input_embeds_warmup_inputs(
*, speculator: MultiModuleMTPSpeculator
) -> dict[str, Any]:
hidden = TritonWarmupTensor(
speculator.dtype,
shape=(1, speculator.hidden_size),
strides=(speculator.hidden_size, 1),
)
return dict(
input_embeds=hidden,
draft_embeds=hidden,
idx_mapping=TritonWarmupTensor(torch.int32),
query_start_loc=TritonWarmupTensor(torch.int32),
last_token_indices=TritonWarmupTensor(torch.int64),
num_reqs=1,
)
@triton_kernel_dispatcher_with_warmup(
kernel=_shift_input_embeds_kernel,
warmup_inputs=_shift_input_embeds_warmup_inputs,
)
def _shift_input_embeds(
input_embeds: torch.Tensor,
draft_embeds: torch.Tensor,
idx_mapping: torch.Tensor,
query_start_loc: torch.Tensor,
last_token_indices: torch.Tensor,
num_reqs: int,
) -> DispatchSpec:
hidden_size = input_embeds.shape[-1]
return (num_reqs, triton.cdiv(hidden_size, 256)), dict(
hidden_size=hidden_size,
BLOCK_SIZE_Q=16,
BLOCK_SIZE_H=256,
)
def update_draft_inputs(
draft_tokens: torch.Tensor,
draft_embeds: torch.Tensor | None,
@@ -1074,29 +1146,21 @@ def update_draft_inputs(
idx_mapping: torch.Tensor,
num_reqs: int,
) -> None:
_shift_input_ids_kernel[(num_reqs,)](
_shift_input_ids(
input_buffers.input_ids,
idx_mapping,
input_buffers.query_start_loc,
last_token_indices,
draft_tokens,
BLOCK_SIZE=1024,
num_reqs=num_reqs,
)
if input_embeds is not None:
assert draft_embeds is not None
hidden_size = input_embeds.shape[-1]
hidden_block_size = 256
_shift_input_embeds_kernel[
(num_reqs, triton.cdiv(hidden_size, hidden_block_size))
](
_shift_input_embeds(
input_embeds,
input_embeds.stride(0),
draft_embeds,
draft_embeds.stride(0),
idx_mapping,
input_buffers.query_start_loc,
last_token_indices,
hidden_size,
BLOCK_SIZE_Q=16,
BLOCK_SIZE_H=hidden_block_size,
num_reqs=num_reqs,
)