[Feat][Perf] Add new warmup infrastructure for JITs (#47451)

Signed-off-by: LopezCastroRoberto <rocastro@redhat.com>
Signed-off-by: Roberto L. Castro <38211239+LopezCastroRoberto@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Roberto L. Castro
2026-07-20 13:21:55 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent fbfe58133d
commit 7ca017778f
14 changed files with 1855 additions and 693 deletions
+238 -70
View File
@@ -14,14 +14,23 @@ preparation.
window indices for sparse prefill.
"""
from dataclasses import dataclass
from typing import Any
import torch
from vllm.model_executor.layers.quantization.utils.quant_utils import (
get_fp8_min_max,
)
from vllm.model_executor.warmup.jit_warmup import VllmJitKernel, zip_inputs
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
TritonPointerInputVariant,
TritonWarmupTensor,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.import_utils import has_cutedsl
from vllm.utils.math_utils import next_power_of_2
@triton.jit
@@ -528,7 +537,6 @@ def combine_topk_swa_indices(
N: int,
) -> tuple[torch.Tensor, torch.Tensor]:
num_tokens = topk_indices.shape[0]
num_reqs = seq_lens.shape[0]
combined_topk = (
(topk + window_size + _SPARSE_PREFILL_TOPK_ALIGNMENT - 1)
// _SPARSE_PREFILL_TOPK_ALIGNMENT
@@ -544,13 +552,10 @@ def combine_topk_swa_indices(
num_tokens, dtype=torch.int32, device=topk_indices.device
)
NUM_WORKERS = 128
_combine_topk_swa_indices_kernel[(num_reqs, NUM_WORKERS)](
_COMBINE_TOPK_SWA_INDICES_KERNEL(
combined_indices,
combined_indices.stride(0),
combined_lens,
topk_indices,
topk_indices.stride(0),
query_start_loc,
seq_lens,
gather_lens,
@@ -559,82 +564,245 @@ def combine_topk_swa_indices(
TOP_K=topk,
COMPRESS_RATIO=compress_ratio,
WINDOW_SIZE=window_size,
PADDED_TOP_K=triton.next_power_of_2(topk_indices.shape[-1]),
)
return combined_indices, combined_lens
@triton.jit
def _combine_topk_swa_indices_kernel(
combined_indices_ptr,
combined_indices_stride,
combined_lens_ptr,
topk_indices_ptr,
topk_indices_stride,
query_start_loc_ptr,
seq_lens_ptr,
gather_lens_ptr,
M,
N,
TOP_K: tl.constexpr,
COMPRESS_RATIO: tl.constexpr,
WINDOW_SIZE: tl.constexpr,
PADDED_TOP_K: tl.constexpr,
_COMBINE_TOPK_SWA_NUM_WORKERS = 128
# Representative pointer alignment variants for Triton pointer specialization.
_COMBINE_TOPK_SWA_POINTER_INPUTS = zip_inputs(
dict(
topk_indices=True,
query_start_loc=True,
seq_lens=True,
gather_lens=True,
),
dict(
topk_indices=True,
query_start_loc=False,
seq_lens=False,
gather_lens=True,
),
dict(
topk_indices=False,
query_start_loc=False,
seq_lens=False,
gather_lens=False,
),
)
_DSV4_COMBINE_TOPK_SWA_WARMUP_INPUTS = zip_inputs(
# DSv4-Flash / SWA-only and C4A.
dict(compress_ratio=1, topk=0, topk_width=512),
dict(compress_ratio=4, topk=512, topk_width=512),
# DSv4-Pro C4A.
dict(compress_ratio=4, topk=1024, topk_width=1024),
# DSv4-Pro C128A.
dict(compress_ratio=128, topk=8192, topk_width=8192),
)
def _hf_config_int(vllm_config: Any, name: str, default: int) -> int:
model_config = getattr(vllm_config, "model_config", None)
hf_config = getattr(model_config, "hf_config", None)
return int(getattr(hf_config, name, default) or default)
def _scheduler_config_int(vllm_config: Any, name: str, default: int) -> int:
scheduler_config = getattr(vllm_config, "scheduler_config", None)
return int(getattr(scheduler_config, name, default) or default)
class CombineTopkSwaIndicesKernel(
VllmJitKernel["CombineTopkSwaIndicesKernel.CompileKey"]
):
batch_idx = tl.program_id(0)
worker_id = tl.program_id(1)
num_workers = tl.num_programs(1)
@dataclass(frozen=True)
class CompileKey:
TOP_K: int
COMPRESS_RATIO: int
WINDOW_SIZE: int
PADDED_TOP_K: int
input_variant: TritonPointerInputVariant
# query_start_loc is a global tensor; rebase to chunk-local offsets
# by subtracting the chunk's starting value.
base = tl.load(query_start_loc_ptr)
query_start = tl.load(query_start_loc_ptr + batch_idx) - base
query_end = tl.load(query_start_loc_ptr + batch_idx + 1) - base
query_len = query_end - query_start
seq_len = tl.load(seq_lens_ptr + batch_idx)
gather_len = tl.load(gather_lens_ptr + batch_idx)
start_pos = seq_len - query_len
# The SWA portion of the gathered buffer starts from position
# (seq_len - gather_len), not position 0. We need this offset
# to correctly index into the gathered buffer.
gather_start = seq_len - gather_len
@staticmethod
@triton.jit(
do_not_specialize=[
"combined_indices_stride",
"topk_indices_stride",
"M",
"N",
]
)
def kernel(
combined_indices_ptr,
combined_indices_stride,
combined_lens_ptr,
topk_indices_ptr,
topk_indices_stride,
query_start_loc_ptr,
seq_lens_ptr,
gather_lens_ptr,
M,
N,
TOP_K: tl.constexpr,
COMPRESS_RATIO: tl.constexpr,
WINDOW_SIZE: tl.constexpr,
PADDED_TOP_K: tl.constexpr,
):
batch_idx = tl.program_id(0)
worker_id = tl.program_id(1)
num_workers = tl.num_programs(1)
for token_idx in range(query_start + worker_id, query_end, num_workers):
# topk_len is fully determined by the query token's absolute position:
# both the C4A indexer and the C128A metadata builder emit
# min((pos + 1) // compress_ratio, topk_tokens) valid entries.
# Caller passes TOP_K=0 for SWA-only layers to zero this out.
token_idx_in_query = token_idx - query_start
pos = start_pos + token_idx_in_query
topk_len = tl.minimum((pos + 1) // COMPRESS_RATIO, TOP_K)
swa_len = tl.minimum(pos + 1, WINDOW_SIZE)
# query_start_loc is a global tensor; rebase to chunk-local offsets
# by subtracting the chunk's starting value.
base = tl.load(query_start_loc_ptr)
query_start = tl.load(query_start_loc_ptr + batch_idx) - base
query_end = tl.load(query_start_loc_ptr + batch_idx + 1) - base
query_len = query_end - query_start
seq_len = tl.load(seq_lens_ptr + batch_idx)
gather_len = tl.load(gather_lens_ptr + batch_idx)
start_pos = seq_len - query_len
# The SWA portion of the gathered buffer starts from position
# (seq_len - gather_len), not position 0. We need this offset
# to correctly index into the gathered buffer.
gather_start = seq_len - gather_len
offset = tl.arange(0, PADDED_TOP_K)
mask = offset < topk_len
topk_indices = tl.load(
topk_indices_ptr + token_idx * topk_indices_stride + offset,
mask=mask,
for token_idx in range(query_start + worker_id, query_end, num_workers):
# topk_len is fully determined by the query token's absolute position:
# both the C4A indexer and the C128A metadata builder emit
# min((pos + 1) // compress_ratio, topk_tokens) valid entries.
# Caller passes TOP_K=0 for SWA-only layers to zero this out.
token_idx_in_query = token_idx - query_start
pos = start_pos + token_idx_in_query
topk_len = tl.minimum((pos + 1) // COMPRESS_RATIO, TOP_K)
swa_len = tl.minimum(pos + 1, WINDOW_SIZE)
offset = tl.arange(0, PADDED_TOP_K)
mask = offset < topk_len
topk_indices = tl.load(
topk_indices_ptr + token_idx * topk_indices_stride + offset,
mask=mask,
)
tl.store(
combined_indices_ptr + token_idx * combined_indices_stride + offset,
topk_indices + M * batch_idx,
mask=mask,
)
offset = tl.arange(0, WINDOW_SIZE)
# Index into gathered buffer: N + (position - gather_start)
# For positions [pos - swa_len + 1, pos], the buffer indices are:
# [N + pos - swa_len + 1 - gather_start, N + pos - gather_start]
tl.store(
combined_indices_ptr
+ token_idx * combined_indices_stride
+ topk_len
+ offset,
M * batch_idx + N + offset + pos - swa_len + 1 - gather_start,
mask=offset < swa_len,
)
combined_len = topk_len + swa_len
tl.store(combined_lens_ptr + token_idx, combined_len)
def dispatch( # type: ignore[override]
self,
*,
topk_width: int,
topk_indices: bool,
query_start_loc: bool,
seq_lens: bool,
gather_lens: bool,
topk: int,
compress_ratio: int,
WINDOW_SIZE: int,
) -> CompileKey:
padded_topk = next_power_of_2(topk_width)
input_variant = TritonPointerInputVariant.from_alignment(
topk_indices=topk_indices,
query_start_loc=query_start_loc,
seq_lens=seq_lens,
gather_lens=gather_lens,
)
tl.store(
combined_indices_ptr + token_idx * combined_indices_stride + offset,
topk_indices + M * batch_idx,
mask=mask,
)
offset = tl.arange(0, WINDOW_SIZE)
# Index into gathered buffer: N + (position - gather_start)
# For positions [pos - swa_len + 1, pos], the buffer indices are:
# [N + pos - swa_len + 1 - gather_start, N + pos - gather_start]
tl.store(
combined_indices_ptr
+ token_idx * combined_indices_stride
+ topk_len
+ offset,
M * batch_idx + N + offset + pos - swa_len + 1 - gather_start,
mask=offset < swa_len,
return self.CompileKey(
TOP_K=topk,
COMPRESS_RATIO=compress_ratio,
WINDOW_SIZE=WINDOW_SIZE,
PADDED_TOP_K=padded_topk,
input_variant=input_variant,
)
combined_len = topk_len + swa_len
tl.store(combined_lens_ptr + token_idx, combined_len)
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
if _scheduler_config_int(vllm_config, "max_num_batched_tokens", 0) <= 0:
return []
window_size = _hf_config_int(vllm_config, "sliding_window", 128)
return self._trace_dispatch(self.dispatch)(
_DSV4_COMBINE_TOPK_SWA_WARMUP_INPUTS,
_COMBINE_TOPK_SWA_POINTER_INPUTS,
WINDOW_SIZE=window_size,
)
def compile(self, compile_key: CompileKey) -> None:
warmup = getattr(self.kernel, "warmup", None)
assert warmup is not None
int32_ptr = TritonWarmupTensor(torch.int32)
input_variant = compile_key.input_variant
warmup(
int32_ptr,
1, # do not specialize combined_indices_stride
int32_ptr,
input_variant.pointer("topk_indices", torch.int32),
1, # do not specialize topk_indices_stride
input_variant.pointer("query_start_loc", torch.int32),
input_variant.pointer("seq_lens", torch.int32),
input_variant.pointer("gather_lens", torch.int32),
1, # do not specialize M
1, # do not specialize N
TOP_K=compile_key.TOP_K,
COMPRESS_RATIO=compile_key.COMPRESS_RATIO,
WINDOW_SIZE=compile_key.WINDOW_SIZE,
PADDED_TOP_K=compile_key.PADDED_TOP_K,
grid=(1, _COMBINE_TOPK_SWA_NUM_WORKERS),
)
def __call__(
self,
combined_indices: torch.Tensor,
combined_lens: torch.Tensor,
topk_indices: torch.Tensor,
query_start_loc: torch.Tensor,
seq_lens: torch.Tensor,
gather_lens: torch.Tensor,
M: int,
N: int,
*,
TOP_K: int,
COMPRESS_RATIO: int,
WINDOW_SIZE: int,
) -> None:
num_reqs = seq_lens.shape[0]
self.kernel[(num_reqs, _COMBINE_TOPK_SWA_NUM_WORKERS)](
combined_indices,
combined_indices.stride(0),
combined_lens,
topk_indices,
topk_indices.stride(0),
query_start_loc,
seq_lens,
gather_lens,
M,
N,
TOP_K=TOP_K,
COMPRESS_RATIO=COMPRESS_RATIO,
WINDOW_SIZE=WINDOW_SIZE,
PADDED_TOP_K=next_power_of_2(topk_indices.shape[-1]),
)
_COMBINE_TOPK_SWA_INDICES_KERNEL = CombineTopkSwaIndicesKernel()
def build_flashinfer_mixed_sparse_indices(