[bug] AsyncScheduler drops first post-resume token after pause_generation + clear_cache (#42117)

Signed-off-by: hao-aaron <ahao@anyscale.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Aaron Hao
2026-05-19 01:06:21 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent b14be81c1f
commit f34623bf3c
4 changed files with 13 additions and 15 deletions
-6
View File
@@ -227,9 +227,6 @@ llm_kwargs = dict(
attention_backend=ATTN_BACKEND,
gpu_memory_utilization=0.75,
weight_transfer_config=WeightTransferConfig(backend="nccl"),
# TODO(haosdent): re-enable once #42043 is fixed. Both LLM
# instances must match.
async_scheduling=False,
)
llm_kwargs.update(rocm_determinism_kwargs)
@@ -371,9 +368,6 @@ llm_v2_kwargs = dict(
gpu_memory_utilization=0.75,
distributed_executor_backend="ray",
attention_backend=ATTN_BACKEND,
# TODO(haosdent): re-enable once #42043 is fixed. Both LLM
# instances must match.
async_scheduling=False,
)
llm_v2_kwargs.update(rocm_determinism_kwargs)
+5 -4
View File
@@ -37,10 +37,11 @@ class AsyncScheduler(Scheduler):
def _update_request_with_output(
self, request: Request, new_token_ids: list[int]
) -> tuple[list[int], bool]:
if request.discard_latest_async_tokens:
# If the request is force preempted in reset_prefix_cache, we
# should discard the latest async token.
request.discard_latest_async_tokens = False
if request.async_tokens_to_discard > 0:
# The request was force-preempted in reset_prefix_cache; drop one
# stale in-flight async output frame per call until the counter
# is drained.
request.async_tokens_to_discard -= 1
return [], False
status_before_update = request.status
+7 -3
View File
@@ -1905,10 +1905,14 @@ class Scheduler(SchedulerInterface):
while self.running:
request = self.running.pop()
self._preempt_request(request, timestamp)
# NOTE(zhuohan): For async scheduling, we need to discard the latest
# output token on the fly to avoid a redundant repetitive output token.
# For async scheduling, any output frames already in flight at
# preemption time are now stale and must be discarded when they
# return. num_output_placeholders is exactly that count: 0 if
# the engine has drained (e.g. pause_generation(keep) waited
# for idle), 1 for vanilla async mid-step, or 1 + spec/PP frames
# otherwise.
request.async_tokens_to_discard = request.num_output_placeholders
request.num_output_placeholders = 0
request.discard_latest_async_tokens = True
# Clear scheduled request ids cache. Since we are forcing preemption
# + resumption in the same step, we must act as if these requests were
+1 -2
View File
@@ -139,8 +139,7 @@ class Request:
# Used in async scheduling.
self.num_output_placeholders = 0
# Used in forced preemption (reset_prefix_cache) with async scheduling.
self.discard_latest_async_tokens = False
self.async_tokens_to_discard = 0
self.spec_token_ids: list[int] = []
self.num_computed_tokens = 0