[Model Runner V2] Implement get_num_sampled_and_rejected kernel (#30029)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
This commit is contained in:
Woosuk Kwon
2025-12-04 07:19:42 -08:00
committed by GitHub
parent 5c32a06a04
commit cc050558f4
3 changed files with 65 additions and 29 deletions
@@ -69,15 +69,3 @@ def rejection_sample(
num_warps=1,
)
return sampled, num_sampled
@torch.compile(dynamic=True)
def get_num_rejected(
cu_num_logits: torch.Tensor,
num_sampled: torch.Tensor,
) -> torch.Tensor:
num_logits = cu_num_logits[1:] - cu_num_logits[:-1]
num_rejected = num_logits - num_sampled
# No token is rejected for chunked prefills.
num_rejected *= num_sampled > 0
return num_rejected