[Kernel] Triton-based Top-k and Top-p sampler kernels (#33538)
Signed-off-by: js_park <cakeng@naver.com> Signed-off-by: Jongseok Park <37990712+cakeng@users.noreply.github.com> Signed-off-by: Sunga Kim <sunga.kim@berkeley.edu> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Sunga Kim <sunga.kim@berkeley.edu> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Sunga Kim
Nick Hill
parent
dc5fa77a4e
commit
c656ba3b4d
@@ -14,16 +14,12 @@ def cdiv(a: int, b: int) -> int:
|
||||
|
||||
def next_power_of_2(n: int) -> int:
|
||||
"""The next power of 2 (inclusive)"""
|
||||
if n < 1:
|
||||
return 1
|
||||
return 1 << (n - 1).bit_length()
|
||||
return 1 if n < 1 else 1 << (n - 1).bit_length()
|
||||
|
||||
|
||||
def prev_power_of_2(n: int) -> int:
|
||||
"""The previous power of 2 (inclusive)"""
|
||||
if n <= 0:
|
||||
return 0
|
||||
return 1 << (n.bit_length() - 1)
|
||||
return 0 if n <= 0 else 1 << (n.bit_length() - 1)
|
||||
|
||||
|
||||
def round_up(x: int, y: int) -> int:
|
||||
|
||||
@@ -11,6 +11,10 @@ from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config.model import LogprobsMode
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
|
||||
if HAS_TRITON:
|
||||
from vllm.v1.sample.ops.topk_topp_triton import apply_top_k_top_p_triton
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -87,8 +91,6 @@ class TopKTopPSampler(nn.Module):
|
||||
else:
|
||||
self.forward = self.forward_native
|
||||
|
||||
self.apply_top_k_top_p = apply_top_k_top_p
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
@@ -101,7 +103,7 @@ class TopKTopPSampler(nn.Module):
|
||||
|
||||
The logits tensor may be updated in-place.
|
||||
"""
|
||||
logits = self.apply_top_k_top_p(logits, k, p)
|
||||
logits = apply_top_k_top_p(logits, k, p)
|
||||
logits_to_return = None
|
||||
if self.logprobs_mode == "processed_logits":
|
||||
logits_to_return = logits
|
||||
@@ -149,7 +151,7 @@ class TopKTopPSampler(nn.Module):
|
||||
|
||||
The logits tensor may be updated in-place.
|
||||
"""
|
||||
logits = self.apply_top_k_top_p(logits, k, p)
|
||||
logits = apply_top_k_top_p_pytorch(logits, k, p, allow_cpu_sync=True)
|
||||
logits_to_return = None
|
||||
if self.logprobs_mode == "processed_logits":
|
||||
logits_to_return = logits
|
||||
@@ -158,14 +160,14 @@ class TopKTopPSampler(nn.Module):
|
||||
|
||||
if len(generators) != logits.shape[0]:
|
||||
return compiled_random_sample(logits), logits_to_return
|
||||
else:
|
||||
probs = logits.softmax(dim=-1, dtype=torch.float32)
|
||||
q = torch.empty_like(probs)
|
||||
q.exponential_()
|
||||
for i, generator in generators.items():
|
||||
q[i].exponential_(generator=generator)
|
||||
|
||||
return probs.div_(q).argmax(dim=-1).view(-1), logits_to_return
|
||||
probs = logits.softmax(dim=-1, dtype=torch.float32)
|
||||
q = torch.empty_like(probs)
|
||||
q.exponential_()
|
||||
for i, generator in generators.items():
|
||||
q[i].exponential_(generator=generator)
|
||||
|
||||
return probs.div_(q).argmax(dim=-1).view(-1), logits_to_return
|
||||
|
||||
def forward_hip(
|
||||
self,
|
||||
@@ -241,9 +243,23 @@ def compiled_random_sample(logits: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
|
||||
def apply_top_k_top_p(
|
||||
logits: torch.Tensor, k: torch.Tensor | None, p: torch.Tensor | None
|
||||
) -> torch.Tensor:
|
||||
if p is None and k is None:
|
||||
return logits
|
||||
|
||||
if HAS_TRITON and logits.shape[0] >= 8:
|
||||
return apply_top_k_top_p_triton(logits, k, p)
|
||||
|
||||
# Use pytorch sort implementation for small batch sizes.
|
||||
return apply_top_k_top_p_pytorch(logits, k, p)
|
||||
|
||||
|
||||
def apply_top_k_top_p_pytorch(
|
||||
logits: torch.Tensor,
|
||||
k: torch.Tensor | None,
|
||||
p: torch.Tensor | None,
|
||||
allow_cpu_sync: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""Apply top-k and top-p masks to the logits.
|
||||
|
||||
@@ -256,8 +272,9 @@ def apply_top_k_top_p(
|
||||
if k is None:
|
||||
return logits
|
||||
|
||||
# Avoid sorting vocab for top-k only case.
|
||||
return apply_top_k_only(logits, k)
|
||||
if allow_cpu_sync:
|
||||
# Avoid sorting vocab for top-k only case.
|
||||
return apply_top_k_only(logits, k)
|
||||
|
||||
logits_sort, logits_idx = logits.sort(dim=-1, descending=False)
|
||||
|
||||
@@ -279,18 +296,16 @@ def apply_top_k_top_p(
|
||||
logits_sort.masked_fill_(top_p_mask, -float("inf"))
|
||||
|
||||
# Re-sort the probabilities.
|
||||
logits = logits_sort.scatter(dim=-1, index=logits_idx, src=logits_sort)
|
||||
return logits
|
||||
return logits.scatter_(dim=-1, index=logits_idx, src=logits_sort)
|
||||
|
||||
|
||||
def apply_top_k_only(
|
||||
logits: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
def apply_top_k_only(logits: torch.Tensor, k: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Apply top-k mask to the logits.
|
||||
|
||||
This implementation doesn't involve sorting the entire vocab.
|
||||
Note however that it involves a GPU->CPU sync which can be detrimental for
|
||||
async scheduling performance.
|
||||
|
||||
The logits tensor may be updated in-place.
|
||||
"""
|
||||
@@ -304,8 +319,7 @@ def apply_top_k_only(
|
||||
top_k_mask = logits.topk(max_top_k, dim=1).values.gather(1, k_index.long())
|
||||
# Handle non-topk rows.
|
||||
top_k_mask.masked_fill_(no_top_k_mask.unsqueeze(1), -float("inf"))
|
||||
logits.masked_fill_(logits < top_k_mask, -float("inf"))
|
||||
return logits
|
||||
return logits.masked_fill_(logits < top_k_mask, -float("inf"))
|
||||
|
||||
|
||||
def random_sample(
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user