[Bugfix][EC Connector] Don't stop an encoder-instance request before its images are encoded (#50275)

Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
Tianyu Guo
2026-08-05 05:53:58 +00:00
committed by GitHub
parent 7794b1e08b
commit 50c51682a1
4 changed files with 121 additions and 5 deletions
+12
View File
@@ -94,6 +94,7 @@ class Scheduler(SchedulerInterface):
)
self.structured_output_manager = structured_output_manager
self.is_encoder_decoder = vllm_config.model_config.is_encoder_decoder
self.is_encoder_only = vllm_config.is_encoder_only
# include_finished_set controls whether a separate set of finished
# request ids should be included in the EngineCoreOutputs returned
@@ -1813,6 +1814,17 @@ class Scheduler(SchedulerInterface):
# Pooling stops as soon as there is output.
request.status = RequestStatus.FINISHED_STOPPED
stopped = True
elif (
self.is_encoder_only
and request.num_computed_tokens >= request.num_prompt_tokens
):
# An encoder instance runs the encoder and publishes the
# embeddings instead of sampling, so it stops as soon as the
# whole prompt is consumed. Encoder inputs are never scheduled
# past a multi-modal item the encoder cache could not admit, so
# a consumed prompt also means every item in it was encoded.
request.status = RequestStatus.FINISHED_STOPPED
stopped = True
if new_token_ids and self.structured_output_manager.should_advance(
request, new_token_ids=new_token_ids
+4 -2
View File
@@ -358,8 +358,10 @@ def make_empty_encoder_model_runner_output(
# Give every request its own contiguous index
req_id_to_index: dict[str, int] = {rid: idx for idx, rid in enumerate(req_ids)}
# No tokens generated yet ⇒ one empty list per request
sampled_token_ids: list[list[int]] = [[0] for _ in req_ids]
# An encoder instance never samples, so it emits no tokens at all. The
# scheduler finishes these requests once their prompt is fully encoded
# (see `Scheduler.update_from_output`).
sampled_token_ids: list[list[int]] = [[] for _ in req_ids]
# Pooler outputs are not available yet ⇒ use None placeholders
pooler_output: list[torch.Tensor | None] = [None for _ in req_ids]