[Model Runner V2] Implement get_num_sampled_and_rejected kernel (#30029)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
This commit is contained in:
@@ -354,6 +354,55 @@ def combine_sampled_and_draft_tokens(
|
||||
return logits_indices
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _get_num_sampled_and_rejected_kernel(
|
||||
num_sampled_ptr,
|
||||
num_rejected_ptr,
|
||||
seq_lens_ptr,
|
||||
cu_num_logits_ptr,
|
||||
idx_mapping_ptr,
|
||||
prefill_len_ptr,
|
||||
):
|
||||
batch_idx = tl.program_id(0)
|
||||
req_state_idx = tl.load(idx_mapping_ptr + batch_idx)
|
||||
|
||||
seq_len = tl.load(seq_lens_ptr + batch_idx)
|
||||
prefill_len = tl.load(prefill_len_ptr + req_state_idx)
|
||||
is_chunked_prefilling = seq_len < prefill_len
|
||||
|
||||
num_sampled = tl.load(num_sampled_ptr + batch_idx)
|
||||
num_sampled = tl.where(is_chunked_prefilling, 0, num_sampled)
|
||||
tl.store(num_sampled_ptr + batch_idx, num_sampled)
|
||||
|
||||
logits_start = tl.load(cu_num_logits_ptr + batch_idx)
|
||||
logits_end = tl.load(cu_num_logits_ptr + batch_idx + 1)
|
||||
num_logits = logits_end - logits_start
|
||||
|
||||
num_rejected = num_logits - num_sampled
|
||||
num_rejected = tl.where(is_chunked_prefilling, 0, num_rejected)
|
||||
tl.store(num_rejected_ptr + batch_idx, num_rejected)
|
||||
|
||||
|
||||
def get_num_sampled_and_rejected(
|
||||
num_sampled: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
cu_num_logits: torch.Tensor,
|
||||
idx_mapping: torch.Tensor,
|
||||
prefill_len: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
num_reqs = idx_mapping.shape[0]
|
||||
num_rejected = torch.empty_like(num_sampled)
|
||||
_get_num_sampled_and_rejected_kernel[(num_reqs,)](
|
||||
num_sampled,
|
||||
num_rejected,
|
||||
seq_lens,
|
||||
cu_num_logits,
|
||||
idx_mapping,
|
||||
prefill_len,
|
||||
)
|
||||
return num_sampled, num_rejected
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _post_update_kernel(
|
||||
idx_mapping_ptr,
|
||||
|
||||
Reference in New Issue
Block a user