[Bugfix] Avoid racy accepted counts in async spec decode (#45100)
Signed-off-by: Weiwei Sun <68775773+sunnweiwei@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Jiangyun Zhu
parent
435f82d61a
commit
cec2ec1176
@@ -2035,7 +2035,13 @@ class GPUModelRunner(
|
||||
|
||||
# Sync num_accepted_tokens from CPU (set by
|
||||
# _update_states_after_model_execute for hybrid models).
|
||||
if self.num_accepted_tokens_event is not None:
|
||||
# Skipped under async scheduling (non-align): the CPU copy races with
|
||||
# the in-flight D2H copy and with input-batch row moves.
|
||||
needs_cpu_accepted_counts = self.num_accepted_tokens_event is not None and not (
|
||||
self.use_async_scheduling and self.cache_config.mamba_cache_mode != "align"
|
||||
)
|
||||
if needs_cpu_accepted_counts:
|
||||
assert self.num_accepted_tokens_event is not None
|
||||
self.num_accepted_tokens_event.synchronize()
|
||||
# Async mode: condense() reordered indices, use prev_positions mapping
|
||||
if self.use_async_scheduling and prev_req_id_to_index:
|
||||
@@ -2058,6 +2064,8 @@ class GPUModelRunner(
|
||||
self.num_accepted_tokens.np[num_reqs:].fill(1)
|
||||
self.num_accepted_tokens.copy_to_gpu()
|
||||
else:
|
||||
# Default to 1; update_num_computed_tokens_for_batch_change below
|
||||
# corrects rows that had drafts from valid_sampled_token_count.
|
||||
self.num_accepted_tokens.np.fill(1)
|
||||
self.num_accepted_tokens.gpu.fill_(1)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user