[Async][Feat] support apply penalty or bad_words for async + spec (#30495)

Signed-off-by: zhuhaoran <zhuhaoran.zhr@alibaba-inc.com>
Signed-off-by: izhuhaoran <izhuhaoran@qq.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
zhrrr
2026-01-09 02:31:50 +00:00
committed by GitHub
co-authored by Nick Hill
parent a4ec0c5595
commit 8ff4a99566
4 changed files with 70 additions and 34 deletions
+34 -3
View File
@@ -965,9 +965,40 @@ class InputBatch:
if sampled_token_ids is None:
assert self.async_copy_ready_event is not None
self.async_copy_ready_event.synchronize()
sampled_token_ids = self.sampled_token_ids_cpu.squeeze(-1).tolist()
# Replace placeholder token id with actual sampled id.
req_output_token_ids[-1] = sampled_token_ids[prev_index]
sampled_token_ids = self.sampled_token_ids_cpu.tolist()
# Replace placeholder token id(s) with actual sampled id(s).
new_ids: list[int] = sampled_token_ids[prev_index]
if not new_ids:
continue
num_sampled_ids = len(new_ids) if new_ids[-1] != -1 else new_ids.index(-1)
# Also account for case where there may be a smaller number of
# output placeholders (tokens can be discarded after a kv-load failure).
first_placeholder = req_output_token_ids.index(-1)
num_placeholders = len(req_output_token_ids) - first_placeholder
num_to_replace = min(num_sampled_ids, num_placeholders)
del new_ids[num_to_replace:]
end_index = first_placeholder + num_to_replace
req_output_token_ids[first_placeholder:end_index] = new_ids
def update_async_spec_token_ids(self, draft_token_ids: list[list[int]]) -> None:
"""
In async scheduling case, update spec_token_ids in sampling metadata with
real draft token ids from prior step. This is called right before they are
needed by the rejection sampler for penalty/bad_words computation.
"""
if not draft_token_ids or not self.prev_req_id_to_index:
return
if (spec_token_ids := self.sampling_metadata.spec_token_ids) is not None:
for req_id, spec_ids in zip(self.req_ids, spec_token_ids):
if spec_ids:
prev_index = self.prev_req_id_to_index.get(req_id)
if prev_index is not None:
draft_ids = draft_token_ids[prev_index]
if draft_ids:
del draft_ids[len(spec_ids) :]
spec_ids.clear()
spec_ids.extend(draft_ids)
@property
def num_reqs(self) -> int: