[6/N][warmup][DSv4] Migrate sampling, and DFlash JIT kernels (#56323)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: LopezCastroRoberto <rocastro@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Roberto L. Castro
2026-09-12 17:24:24 -07:00
committed by GitHub
co-authored by Lucas Wilkinson OpenAI Codex
parent e19a3e172e
commit aed894c190
27 changed files with 2288 additions and 558 deletions
+20 -1
View File
@@ -13,11 +13,27 @@ from vllm.platforms import CpuArchEnum, current_platform
from vllm.triton_utils import HAS_TRITON
if HAS_TRITON:
from vllm.v1.sample.ops.topk_topp_triton import apply_top_k_top_p_triton
from vllm.v1.sample.ops.topk_topp_triton import (
_topk_topp,
_topp_split_mask,
_topp_split_stats,
_topp_split_step,
apply_top_k_top_p_triton,
)
logger = init_logger(__name__)
def register_top_k_top_p_warmups() -> None:
"""Register every native accelerator sampling kernel used at runtime."""
if HAS_TRITON and not current_platform.is_cpu():
_topk_topp.register_warmup()
if current_platform.is_cuda_alike():
_topp_split_stats.register_warmup()
_topp_split_step.register_warmup()
_topp_split_mask.register_warmup()
def _skip_aiter_sampler_on_gfx1250() -> bool:
# Lazy ROCm-only import; keeps arch detection out of import time on CUDA/CPU.
from vllm.platforms.rocm import on_gfx1250
@@ -128,6 +144,9 @@ class TopKTopPSampler(nn.Module):
else:
self.forward = self.forward_native
# Every accelerator backend can fall back to native sampling at runtime.
register_top_k_top_p_warmups()
def forward_native(
self,
logits: torch.Tensor,
+267 -66
View File
@@ -9,8 +9,21 @@ using Pivot-based Truncation and Selection" By Park et al.
"""
from typing import Any
import torch
from vllm.model_executor.warmup.jit_warmup import (
WarmupChoices,
WarmupIntRange,
_when,
)
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
DispatchSpec,
TritonWarmupTensor,
triton_kernel_dispatcher_with_warmup,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.math_utils import next_power_of_2
@@ -863,6 +876,90 @@ def _topk_topp_kernel(
tl.store(LOGITS_ROW + offs_n, logits_blk, mask=mask_n)
def _max_sampler_batch_size(vllm_config: Any) -> int:
return vllm_config.scheduler_config.max_num_seqs * (
vllm_config.num_speculative_tokens + 1
)
def _topk_topp_warmup_inputs(vllm_config: Any) -> dict[str, Any]:
vocab_size = vllm_config.model_config.get_vocab_size()
split_enabled = current_platform.is_cuda_alike()
mode: Any = WarmupChoices((True, False), (True, True), (False, True))
logits_stride: Any = WarmupChoices(16, 2)
batch_size: Any = WarmupIntRange(1, _max_sampler_batch_size(vllm_config) + 1)
topk_enabled = mode[0]
topp_enabled = mode[1]
_when(topk_enabled or not (split_enabled and batch_size <= _SPLIT_MAX_BATCH))
logits = TritonWarmupTensor(
torch.float32,
shape=(batch_size, vocab_size),
strides=(logits_stride, 1),
)
return dict(
logits=logits,
buffer=TritonWarmupTensor(torch.float32),
percentile_to_std_table=TritonWarmupTensor(torch.float32),
normal_cdf_to_sigma_table=TritonWarmupTensor(torch.float32),
k=TritonWarmupTensor(torch.int32) if topk_enabled else None,
p=TritonWarmupTensor(torch.float32) if topp_enabled else None,
mask_value=float("-inf"),
num_sm=num_compute_units(),
)
@triton_kernel_dispatcher_with_warmup(
kernel=_topk_topp_kernel,
warmup_inputs=_topk_topp_warmup_inputs,
)
def _topk_topp(
logits: torch.Tensor,
buffer: torch.Tensor,
percentile_to_std_table: torch.Tensor,
normal_cdf_to_sigma_table: torch.Tensor,
k: torch.Tensor | None,
p: torch.Tensor | None,
mask_value: float,
num_sm: int,
) -> DispatchSpec:
batch_size, vocab_size = logits.shape
topk_enabled = k is not None
topp_enabled = p is not None
k_ptr = k if k is not None else logits
p_ptr = p if p is not None else logits
num_programs = min(num_sm, batch_size)
# Smaller tiles compile and run faster on CPU; GPU benefits from larger tiles.
# On XPU, large BLOCK_SIZE causes precision loss in the single-pass pivot
# approximation; use smaller tiles for accurate top-p results.
if logits.device.type == "cpu":
block_size, block_size_trunc, num_warps = 256, 128, 4
elif logits.device.type == "xpu":
block_size, block_size_trunc, num_warps = 4096, 2048, 4
else:
block_size, block_size_trunc, num_warps = 8192, 4096, 8
# Each program serially sweeps the vocab row in BLOCK_SIZE tiles, so
# per-tile latency bounds kernel latency, and Triton's default of 4
# warps leaves an 8192-wide tile at 16 elements per lane. 8 warps is
# faster on every arch measured (SM90, SM100, SM120, gfx950); 16 is not.
split_covers_ponly = (
current_platform.is_cuda_alike()
and topp_enabled
and batch_size <= _SPLIT_MAX_BATCH
)
return (num_programs,), dict(
K=k_ptr,
P=p_ptr,
BATCH_SIZE=batch_size,
VOCAB_SIZE=vocab_size,
BLOCK_SIZE=block_size,
BLOCK_SIZE_TRUNC=block_size_trunc,
TOPK_ENABLED=topk_enabled,
TOPP_ENABLED=topp_enabled,
SPLIT_COVERS_PONLY=split_covers_ponly,
num_warps=num_warps,
)
# ---------------------------------------------------------------------------
# Split-row pipeline for small batches.
#
@@ -892,6 +989,13 @@ _SPLIT_ROUNDS = 5
_TRITON_SPLIT_CACHE: dict[torch.device, dict[str, torch.Tensor]] = {}
def _topp_split_count(batch_size: int, num_sm: int) -> int:
splits = 1
while splits < _SPLIT_MAX_SPLITS and splits * 2 * batch_size <= num_sm:
splits *= 2
return splits
@triton.jit
def _topp_sb_stats_kernel(
LOGITS,
@@ -939,6 +1043,51 @@ def _topp_sb_stats_kernel(
tl.store(STATS + base + 3, cnt)
def _topp_split_stats_warmup_inputs(vllm_config: Any) -> dict[str, Any]:
vocab_size = vllm_config.model_config.get_vocab_size()
num_sm = num_compute_units()
logits_stride: Any = WarmupChoices(16, 2)
batch_size: Any = WarmupIntRange(
1, min(_max_sampler_batch_size(vllm_config), _SPLIT_MAX_BATCH) + 1
)
has_k: Any = WarmupChoices(False, True)
logits = TritonWarmupTensor(
torch.float32,
shape=(batch_size, vocab_size),
strides=(logits_stride, 1),
)
return dict(
logits=logits,
stats=TritonWarmupTensor(torch.float32),
k=TritonWarmupTensor(torch.int32) if has_k else None,
p=TritonWarmupTensor(torch.float32),
num_sm=num_sm,
)
@triton_kernel_dispatcher_with_warmup(
kernel=_topp_sb_stats_kernel,
warmup_inputs=_topp_split_stats_warmup_inputs,
)
def _topp_split_stats(
logits: torch.Tensor,
stats: torch.Tensor,
k: torch.Tensor | None,
p: torch.Tensor,
num_sm: int,
) -> DispatchSpec:
batch_size, vocab_size = logits.shape
splits = _topp_split_count(batch_size, num_sm)
return (logits.shape[0] * splits,), dict(
K=k if k is not None else logits,
HAS_K=k is not None,
VOCAB_SIZE=vocab_size,
S=splits,
BLOCK=8192,
num_warps=8,
)
@triton.jit
def _topp_sb_row_stats(STATS, row, S: tl.constexpr):
"""Combine per-slice partials into row (max, sum_exp, min, n_finite)."""
@@ -1183,6 +1332,58 @@ def _topp_sb_step_kernel(
tl.store(pbase + fidx * 3 + 2, nmin)
def _topp_split_step_warmup_inputs(vllm_config: Any) -> dict[str, Any]:
vocab_size = vllm_config.model_config.get_vocab_size()
num_sm = num_compute_units()
logits_stride: Any = WarmupChoices(16, 2)
batch_size: Any = WarmupIntRange(
1, min(_max_sampler_batch_size(vllm_config), _SPLIT_MAX_BATCH) + 1
)
round: Any = WarmupIntRange(0, _SPLIT_ROUNDS)
has_k: Any = WarmupChoices(False, True)
logits = TritonWarmupTensor(
torch.float32,
shape=(batch_size, vocab_size),
strides=(logits_stride, 1),
)
return dict(
logits=logits,
stats=TritonWarmupTensor(torch.float32),
parts=TritonWarmupTensor(torch.float32),
k=TritonWarmupTensor(torch.int32) if has_k else None,
p=TritonWarmupTensor(torch.float32),
round=round,
num_sm=num_sm,
)
@triton_kernel_dispatcher_with_warmup(
kernel=_topp_sb_step_kernel,
warmup_inputs=_topp_split_step_warmup_inputs,
)
def _topp_split_step(
logits: torch.Tensor,
stats: torch.Tensor,
parts: torch.Tensor,
k: torch.Tensor | None,
p: torch.Tensor,
round: int,
num_sm: int,
) -> DispatchSpec:
batch_size, vocab_size = logits.shape
splits = _topp_split_count(batch_size, num_sm)
return (logits.shape[0] * splits,), dict(
K=k if k is not None else logits,
HAS_K=k is not None,
S=splits,
F=_SPLIT_FANOUT,
NUM_ROUNDS=_SPLIT_ROUNDS,
VOCAB_SIZE=vocab_size,
BLOCK=2048,
num_warps=8,
)
@triton.jit
def _topp_sb_mask_kernel(
LOGITS,
@@ -1313,6 +1514,57 @@ def _topp_sb_mask_kernel(
tl.store(ROW + i + offs, tl.where(keep, x, MASK_VALUE), mask=mask_n)
def _topp_split_mask_warmup_inputs(vllm_config: Any) -> dict[str, Any]:
vocab_size = vllm_config.model_config.get_vocab_size()
num_sm = num_compute_units()
logits_stride: Any = WarmupChoices(16, 2)
batch_size: Any = WarmupIntRange(
1, min(_max_sampler_batch_size(vllm_config), _SPLIT_MAX_BATCH) + 1
)
has_k: Any = WarmupChoices(False, True)
logits = TritonWarmupTensor(
torch.float32,
shape=(batch_size, vocab_size),
strides=(logits_stride, 1),
)
return dict(
logits=logits,
stats=TritonWarmupTensor(torch.float32),
parts=TritonWarmupTensor(torch.float32),
k=TritonWarmupTensor(torch.int32) if has_k else None,
p=TritonWarmupTensor(torch.float32),
mask_value=float("-inf"),
num_sm=num_sm,
)
@triton_kernel_dispatcher_with_warmup(
kernel=_topp_sb_mask_kernel,
warmup_inputs=_topp_split_mask_warmup_inputs,
)
def _topp_split_mask(
logits: torch.Tensor,
stats: torch.Tensor,
parts: torch.Tensor,
k: torch.Tensor | None,
p: torch.Tensor,
mask_value: float,
num_sm: int,
) -> DispatchSpec:
batch_size, vocab_size = logits.shape
splits = _topp_split_count(batch_size, num_sm)
return (logits.shape[0] * splits,), dict(
K=k if k is not None else logits,
HAS_K=k is not None,
S=splits,
F=_SPLIT_FANOUT,
NUM_ROUNDS=_SPLIT_ROUNDS,
VOCAB_SIZE=vocab_size,
BLOCK=8192,
num_warps=8,
)
def _apply_topp_split(
logits: torch.Tensor,
k: torch.Tensor | None,
@@ -1321,11 +1573,6 @@ def _apply_topp_split(
num_sm: int,
) -> None:
"""Split-row top-p pipeline for small batches; masks logits in place."""
batch_size, vocab_size = logits.shape
splits = 1
while splits < _SPLIT_MAX_SPLITS and splits * 2 * batch_size <= num_sm:
splits *= 2
ws = _TRITON_SPLIT_CACHE.get(logits.device)
if ws is None:
ws = {
@@ -1336,53 +1583,31 @@ def _apply_topp_split(
}
_TRITON_SPLIT_CACHE[logits.device] = ws
k_ptr = k if k is not None else logits # dummy pointer when HAS_K=False
has_k = k is not None
_topp_sb_stats_kernel[(batch_size * splits,)](
_topp_split_stats(
logits,
logits.stride(0),
ws["stats"],
k_ptr,
k,
p,
HAS_K=has_k,
VOCAB_SIZE=vocab_size,
S=splits,
BLOCK=8192,
num_warps=8,
num_sm,
)
for round_i in range(_SPLIT_ROUNDS):
_topp_sb_step_kernel[(batch_size * splits,)](
_topp_split_step(
logits,
logits.stride(0),
ws["stats"],
ws["parts"],
k_ptr,
k,
p,
round_i,
HAS_K=has_k,
S=splits,
F=_SPLIT_FANOUT,
NUM_ROUNDS=_SPLIT_ROUNDS,
VOCAB_SIZE=vocab_size,
BLOCK=2048,
num_warps=8,
num_sm,
)
_topp_sb_mask_kernel[(batch_size * splits,)](
_topp_split_mask(
logits,
logits.stride(0),
ws["stats"],
ws["parts"],
k_ptr,
k,
p,
HAS_K=has_k,
MASK_VALUE=mask_value,
S=splits,
F=_SPLIT_FANOUT,
NUM_ROUNDS=_SPLIT_ROUNDS,
VOCAB_SIZE=vocab_size,
BLOCK=8192,
num_warps=8,
mask_value,
num_sm,
)
@@ -1477,39 +1702,15 @@ def apply_top_k_top_p_triton(
else:
normal_cdf_to_sigma_table, percentile_to_std_table = tables
# Smaller tiles compile and run faster on CPU; GPU benefits from larger tiles.
# On XPU, large BLOCK_SIZE causes precision loss in the single-pass pivot
# approximation; use smaller tiles for accurate top-p results.
launch_kwargs = {}
if logits.device.type == "cpu":
block_size, block_size_trunc = 256, 128
elif logits.device.type == "xpu":
block_size, block_size_trunc = 4096, 2048
else:
block_size, block_size_trunc = 8192, 4096
# Each program serially sweeps the vocab row in BLOCK_SIZE tiles, so
# per-tile latency bounds kernel latency, and Triton's default of 4
# warps leaves an 8192-wide tile at 16 elements per lane. 8 warps is
# faster on every arch measured (SM90, SM100, SM120, gfx950); 16 is not.
launch_kwargs["num_warps"] = 8
_topk_topp_kernel[(NUM_PROGRAMS,)](
_topk_topp(
logits,
logits.stride(0),
buffer,
percentile_to_std_table,
normal_cdf_to_sigma_table,
k_ptr,
p_ptr,
BATCH_SIZE=batch_size,
MASK_VALUE=mask_value,
VOCAB_SIZE=vocab_size,
BLOCK_SIZE=block_size,
BLOCK_SIZE_TRUNC=block_size_trunc,
TOPK_ENABLED=topk_enabled,
TOPP_ENABLED=topp_enabled,
SPLIT_COVERS_PONLY=use_split,
**launch_kwargs,
k_ptr if topk_enabled else None,
p_ptr if topp_enabled else None,
mask_value,
num_sm,
)
if use_split:
_apply_topp_split(logits, k_ptr, p_ptr, mask_value, num_sm)