[Model Runner V2] support apply penalty for spec decode (#33251)
Signed-off-by: zhuhaoran <zhuhaoran.zhr@alibaba-inc.com>
This commit is contained in:
@@ -40,6 +40,8 @@ class InputBatch:
|
||||
idx_mapping_np: np.ndarray
|
||||
# Identical to idx_mapping except for spec decoding.
|
||||
expanded_idx_mapping: torch.Tensor
|
||||
# [total_num_logits] position within request for each logit
|
||||
expanded_local_pos: torch.Tensor
|
||||
|
||||
# [num_reqs]
|
||||
# batch_idx -> num_scheduled_tokens
|
||||
@@ -91,6 +93,7 @@ class InputBatch:
|
||||
idx_mapping_np = np.arange(num_reqs, dtype=np.int32)
|
||||
idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=device)
|
||||
expanded_idx_mapping = idx_mapping
|
||||
expanded_local_pos = torch.zeros(num_reqs, dtype=torch.int32, device=device)
|
||||
num_scheduled_tokens = np.full(num_reqs, num_tokens // num_reqs, dtype=np.int32)
|
||||
num_scheduled_tokens[-1] += num_tokens % num_reqs
|
||||
assert int(num_scheduled_tokens.sum()) == num_tokens
|
||||
@@ -126,6 +129,7 @@ class InputBatch:
|
||||
idx_mapping=idx_mapping,
|
||||
idx_mapping_np=idx_mapping_np,
|
||||
expanded_idx_mapping=expanded_idx_mapping,
|
||||
expanded_local_pos=expanded_local_pos,
|
||||
num_scheduled_tokens=num_scheduled_tokens,
|
||||
num_tokens=num_tokens,
|
||||
num_tokens_after_padding=num_tokens,
|
||||
@@ -487,6 +491,7 @@ def post_update(
|
||||
def _expand_idx_mapping_kernel(
|
||||
idx_mapping_ptr,
|
||||
expanded_idx_mapping_ptr,
|
||||
expanded_local_pos_ptr,
|
||||
cu_num_logits_ptr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
@@ -499,6 +504,7 @@ def _expand_idx_mapping_kernel(
|
||||
mask = block < num_tokens
|
||||
req_state_idx = tl.load(idx_mapping_ptr + req_idx)
|
||||
tl.store(expanded_idx_mapping_ptr + start_idx + block, req_state_idx, mask=mask)
|
||||
tl.store(expanded_local_pos_ptr + start_idx + block, block, mask=mask)
|
||||
|
||||
|
||||
def expand_idx_mapping(
|
||||
@@ -506,13 +512,17 @@ def expand_idx_mapping(
|
||||
total_num_logits: int,
|
||||
cu_num_logits: torch.Tensor,
|
||||
max_expand_len: int,
|
||||
) -> torch.Tensor:
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
num_reqs = idx_mapping.shape[0]
|
||||
expanded_idx_mapping = idx_mapping.new_empty(total_num_logits)
|
||||
expanded_local_pos = torch.empty(
|
||||
total_num_logits, dtype=torch.int32, device=idx_mapping.device
|
||||
)
|
||||
_expand_idx_mapping_kernel[(num_reqs,)](
|
||||
idx_mapping,
|
||||
expanded_idx_mapping,
|
||||
expanded_local_pos,
|
||||
cu_num_logits,
|
||||
BLOCK_SIZE=triton.next_power_of_2(max_expand_len),
|
||||
)
|
||||
return expanded_idx_mapping
|
||||
return expanded_idx_mapping, expanded_local_pos
|
||||
|
||||
Reference in New Issue
Block a user