[Kernel] Triton-based Top-k and Top-p sampler kernels (#33538)

Signed-off-by: js_park <cakeng@naver.com>
Signed-off-by: Jongseok Park <37990712+cakeng@users.noreply.github.com>
Signed-off-by: Sunga Kim <sunga.kim@berkeley.edu>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Sunga Kim <sunga.kim@berkeley.edu>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Jongseok Park
2026-02-17 23:14:30 +00:00
committed by GitHub
co-authored by Sunga Kim Nick Hill
parent dc5fa77a4e
commit c656ba3b4d
6 changed files with 2002 additions and 32 deletions
+2 -6
View File
@@ -14,16 +14,12 @@ def cdiv(a: int, b: int) -> int:
def next_power_of_2(n: int) -> int:
"""The next power of 2 (inclusive)"""
if n < 1:
return 1
return 1 << (n - 1).bit_length()
return 1 if n < 1 else 1 << (n - 1).bit_length()
def prev_power_of_2(n: int) -> int:
"""The previous power of 2 (inclusive)"""
if n <= 0:
return 0
return 1 << (n.bit_length() - 1)
return 0 if n <= 0 else 1 << (n.bit_length() - 1)
def round_up(x: int, y: int) -> int:
+35 -21
View File
@@ -11,6 +11,10 @@ from vllm._aiter_ops import rocm_aiter_ops
from vllm.config.model import LogprobsMode
from vllm.logger import init_logger
from vllm.platforms import CpuArchEnum, current_platform
from vllm.triton_utils import HAS_TRITON
if HAS_TRITON:
from vllm.v1.sample.ops.topk_topp_triton import apply_top_k_top_p_triton
logger = init_logger(__name__)
@@ -87,8 +91,6 @@ class TopKTopPSampler(nn.Module):
else:
self.forward = self.forward_native
self.apply_top_k_top_p = apply_top_k_top_p
def forward_native(
self,
logits: torch.Tensor,
@@ -101,7 +103,7 @@ class TopKTopPSampler(nn.Module):
The logits tensor may be updated in-place.
"""
logits = self.apply_top_k_top_p(logits, k, p)
logits = apply_top_k_top_p(logits, k, p)
logits_to_return = None
if self.logprobs_mode == "processed_logits":
logits_to_return = logits
@@ -149,7 +151,7 @@ class TopKTopPSampler(nn.Module):
The logits tensor may be updated in-place.
"""
logits = self.apply_top_k_top_p(logits, k, p)
logits = apply_top_k_top_p_pytorch(logits, k, p, allow_cpu_sync=True)
logits_to_return = None
if self.logprobs_mode == "processed_logits":
logits_to_return = logits
@@ -158,14 +160,14 @@ class TopKTopPSampler(nn.Module):
if len(generators) != logits.shape[0]:
return compiled_random_sample(logits), logits_to_return
else:
probs = logits.softmax(dim=-1, dtype=torch.float32)
q = torch.empty_like(probs)
q.exponential_()
for i, generator in generators.items():
q[i].exponential_(generator=generator)
return probs.div_(q).argmax(dim=-1).view(-1), logits_to_return
probs = logits.softmax(dim=-1, dtype=torch.float32)
q = torch.empty_like(probs)
q.exponential_()
for i, generator in generators.items():
q[i].exponential_(generator=generator)
return probs.div_(q).argmax(dim=-1).view(-1), logits_to_return
def forward_hip(
self,
@@ -241,9 +243,23 @@ def compiled_random_sample(logits: torch.Tensor) -> torch.Tensor:
def apply_top_k_top_p(
logits: torch.Tensor, k: torch.Tensor | None, p: torch.Tensor | None
) -> torch.Tensor:
if p is None and k is None:
return logits
if HAS_TRITON and logits.shape[0] >= 8:
return apply_top_k_top_p_triton(logits, k, p)
# Use pytorch sort implementation for small batch sizes.
return apply_top_k_top_p_pytorch(logits, k, p)
def apply_top_k_top_p_pytorch(
logits: torch.Tensor,
k: torch.Tensor | None,
p: torch.Tensor | None,
allow_cpu_sync: bool = False,
) -> torch.Tensor:
"""Apply top-k and top-p masks to the logits.
@@ -256,8 +272,9 @@ def apply_top_k_top_p(
if k is None:
return logits
# Avoid sorting vocab for top-k only case.
return apply_top_k_only(logits, k)
if allow_cpu_sync:
# Avoid sorting vocab for top-k only case.
return apply_top_k_only(logits, k)
logits_sort, logits_idx = logits.sort(dim=-1, descending=False)
@@ -279,18 +296,16 @@ def apply_top_k_top_p(
logits_sort.masked_fill_(top_p_mask, -float("inf"))
# Re-sort the probabilities.
logits = logits_sort.scatter(dim=-1, index=logits_idx, src=logits_sort)
return logits
return logits.scatter_(dim=-1, index=logits_idx, src=logits_sort)
def apply_top_k_only(
logits: torch.Tensor,
k: torch.Tensor,
) -> torch.Tensor:
def apply_top_k_only(logits: torch.Tensor, k: torch.Tensor) -> torch.Tensor:
"""
Apply top-k mask to the logits.
This implementation doesn't involve sorting the entire vocab.
Note however that it involves a GPU->CPU sync which can be detrimental for
async scheduling performance.
The logits tensor may be updated in-place.
"""
@@ -304,8 +319,7 @@ def apply_top_k_only(
top_k_mask = logits.topk(max_top_k, dim=1).values.gather(1, k_index.long())
# Handle non-topk rows.
top_k_mask.masked_fill_(no_top_k_mask.unsqueeze(1), -float("inf"))
logits.masked_fill_(logits < top_k_mask, -float("inf"))
return logits
return logits.masked_fill_(logits < top_k_mask, -float("inf"))
def random_sample(
File diff suppressed because it is too large Load Diff