Fix the order of _free_encoder_inputs (#38907)

Signed-off-by: Tianyu Guo <guoty9@mail2.sysu.edu.cn>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Tianyu Guo
2026-04-10 22:47:48 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 0f3ce4c74b
commit 8d0aabdde9
+4 -8
View File
@@ -996,14 +996,6 @@ class Scheduler(SchedulerInterface):
request.use_structured_output and not request.is_prefill_chunk
)
# NOTE: _free_encoder_inputs relies on num_computed_tokens, which
# may be updated again in _update_from_output for speculative
# decoding. However, it is safe to call the method here because
# encoder inputs are always part of the prompt, not the output,
# and thus are unaffected by speculative decoding.
if request.has_encoder_inputs:
self._free_encoder_inputs(request)
# Clear the finished request IDs.
# NOTE: We shouldn't do self.finished_req_ids.clear() here because
# it will also affect the scheduler output.
@@ -1389,6 +1381,10 @@ class Scheduler(SchedulerInterface):
request_id=req_id,
)
# Free encoder inputs only after the step has actually executed.
if request.has_encoder_inputs:
self._free_encoder_inputs(request)
stopped = False
new_logprobs = None
new_token_ids = generated_token_ids