[Bugfix][EC Connector] Don't stop an encoder-instance request before its images are encoded (#50275)
Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
@@ -94,6 +94,7 @@ class Scheduler(SchedulerInterface):
|
||||
)
|
||||
self.structured_output_manager = structured_output_manager
|
||||
self.is_encoder_decoder = vllm_config.model_config.is_encoder_decoder
|
||||
self.is_encoder_only = vllm_config.is_encoder_only
|
||||
|
||||
# include_finished_set controls whether a separate set of finished
|
||||
# request ids should be included in the EngineCoreOutputs returned
|
||||
@@ -1813,6 +1814,17 @@ class Scheduler(SchedulerInterface):
|
||||
# Pooling stops as soon as there is output.
|
||||
request.status = RequestStatus.FINISHED_STOPPED
|
||||
stopped = True
|
||||
elif (
|
||||
self.is_encoder_only
|
||||
and request.num_computed_tokens >= request.num_prompt_tokens
|
||||
):
|
||||
# An encoder instance runs the encoder and publishes the
|
||||
# embeddings instead of sampling, so it stops as soon as the
|
||||
# whole prompt is consumed. Encoder inputs are never scheduled
|
||||
# past a multi-modal item the encoder cache could not admit, so
|
||||
# a consumed prompt also means every item in it was encoded.
|
||||
request.status = RequestStatus.FINISHED_STOPPED
|
||||
stopped = True
|
||||
|
||||
if new_token_ids and self.structured_output_manager.should_advance(
|
||||
request, new_token_ids=new_token_ids
|
||||
|
||||
+4
-2
@@ -358,8 +358,10 @@ def make_empty_encoder_model_runner_output(
|
||||
# Give every request its own contiguous index
|
||||
req_id_to_index: dict[str, int] = {rid: idx for idx, rid in enumerate(req_ids)}
|
||||
|
||||
# No tokens generated yet ⇒ one empty list per request
|
||||
sampled_token_ids: list[list[int]] = [[0] for _ in req_ids]
|
||||
# An encoder instance never samples, so it emits no tokens at all. The
|
||||
# scheduler finishes these requests once their prompt is fully encoded
|
||||
# (see `Scheduler.update_from_output`).
|
||||
sampled_token_ids: list[list[int]] = [[] for _ in req_ids]
|
||||
|
||||
# Pooler outputs are not available yet ⇒ use None placeholders
|
||||
pooler_output: list[torch.Tensor | None] = [None for _ in req_ids]
|
||||
|
||||
Reference in New Issue
Block a user