[Model Runner V2] Implement get_num_sampled_and_rejected kernel (#30029)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
This commit is contained in:
@@ -69,15 +69,3 @@ def rejection_sample(
|
||||
num_warps=1,
|
||||
)
|
||||
return sampled, num_sampled
|
||||
|
||||
|
||||
@torch.compile(dynamic=True)
|
||||
def get_num_rejected(
|
||||
cu_num_logits: torch.Tensor,
|
||||
num_sampled: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
num_logits = cu_num_logits[1:] - cu_num_logits[:-1]
|
||||
num_rejected = num_logits - num_sampled
|
||||
# No token is rejected for chunked prefills.
|
||||
num_rejected *= num_sampled > 0
|
||||
return num_rejected
|
||||
|
||||
Reference in New Issue
Block a user