[Async][Feat] support apply penalty or bad_words for async + spec (#30495)
Signed-off-by: zhuhaoran <zhuhaoran.zhr@alibaba-inc.com> Signed-off-by: izhuhaoran <izhuhaoran@qq.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -965,9 +965,40 @@ class InputBatch:
|
||||
if sampled_token_ids is None:
|
||||
assert self.async_copy_ready_event is not None
|
||||
self.async_copy_ready_event.synchronize()
|
||||
sampled_token_ids = self.sampled_token_ids_cpu.squeeze(-1).tolist()
|
||||
# Replace placeholder token id with actual sampled id.
|
||||
req_output_token_ids[-1] = sampled_token_ids[prev_index]
|
||||
sampled_token_ids = self.sampled_token_ids_cpu.tolist()
|
||||
# Replace placeholder token id(s) with actual sampled id(s).
|
||||
new_ids: list[int] = sampled_token_ids[prev_index]
|
||||
if not new_ids:
|
||||
continue
|
||||
num_sampled_ids = len(new_ids) if new_ids[-1] != -1 else new_ids.index(-1)
|
||||
# Also account for case where there may be a smaller number of
|
||||
# output placeholders (tokens can be discarded after a kv-load failure).
|
||||
first_placeholder = req_output_token_ids.index(-1)
|
||||
num_placeholders = len(req_output_token_ids) - first_placeholder
|
||||
num_to_replace = min(num_sampled_ids, num_placeholders)
|
||||
del new_ids[num_to_replace:]
|
||||
end_index = first_placeholder + num_to_replace
|
||||
req_output_token_ids[first_placeholder:end_index] = new_ids
|
||||
|
||||
def update_async_spec_token_ids(self, draft_token_ids: list[list[int]]) -> None:
|
||||
"""
|
||||
In async scheduling case, update spec_token_ids in sampling metadata with
|
||||
real draft token ids from prior step. This is called right before they are
|
||||
needed by the rejection sampler for penalty/bad_words computation.
|
||||
"""
|
||||
if not draft_token_ids or not self.prev_req_id_to_index:
|
||||
return
|
||||
|
||||
if (spec_token_ids := self.sampling_metadata.spec_token_ids) is not None:
|
||||
for req_id, spec_ids in zip(self.req_ids, spec_token_ids):
|
||||
if spec_ids:
|
||||
prev_index = self.prev_req_id_to_index.get(req_id)
|
||||
if prev_index is not None:
|
||||
draft_ids = draft_token_ids[prev_index]
|
||||
if draft_ids:
|
||||
del draft_ids[len(spec_ids) :]
|
||||
spec_ids.clear()
|
||||
spec_ids.extend(draft_ids)
|
||||
|
||||
@property
|
||||
def num_reqs(self) -> int:
|
||||
|
||||
Reference in New Issue
Block a user