diff --git a/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py b/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py index b5f99683c2..dab9677295 100644 --- a/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py +++ b/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py @@ -108,8 +108,8 @@ async def fanout_encoder_primer( "messages": [ {"role": "user", "content": [item]}, ], - # Only need 1 token so the server actually runs the encoder path - "max_tokens": 1, + # No max_tokens cap: the encoder instance never samples, it finishes + # once the prompt is encoded and its embeddings are published. "stream": False, } tasks.append( diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index ac966ad11b..0acfef1e60 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -38,7 +38,12 @@ from vllm.v1.kv_cache_interface import ( KVCacheGroupSpec, MambaSpec, ) -from vllm.v1.outputs import DraftTokenIds, KVConnectorOutput, ModelRunnerOutput +from vllm.v1.outputs import ( + DraftTokenIds, + KVConnectorOutput, + ModelRunnerOutput, + make_empty_encoder_model_runner_output, +) from vllm.v1.request import Request, RequestStatus from vllm.v1.structured_output import StructuredOutputGrammar, StructuredOutputManager @@ -5730,3 +5735,100 @@ def test_hybrid_per_group_hit_divergence_fa_deeper_no_external(): num_scheduled = output.num_scheduled_tokens[replay.request_id] # Must resume at the convergent boundary (block 0), not the deep FA hit. assert replay.num_tokens - num_scheduled == block_size + + +def _make_encoder_instance_request(scheduler, text_prefix=8, image_tokens=16): + """One request whose image sits after some text, as an EPD proxy sends it. + + `create_scheduler` builds a text-only model, so `compute_mm_encoder_budget` + leaves the encoder cache at zero capacity; give it room so the admitted path + is reachable. + """ + ecm = scheduler.encoder_cache_manager + ecm.cache_size = ecm.num_free_slots = ecm.num_freeable_slots = 8 * image_tokens + scheduler.max_num_encoder_input_tokens = 8 * image_tokens + + (request,) = create_requests( + num_requests=1, + num_tokens=text_prefix + image_tokens + 4, + # Completion must hinge on the prompt being encoded, not on max_tokens. + max_tokens=16, + mm_hashes_list=[["img-hash-0"]], + mm_positions=[[PlaceholderRange(offset=text_prefix, length=image_tokens)]], + ) + scheduler.add_request(request) + return request + + +def test_encoder_instance_defers_stop_until_prompt_is_consumed(): + """An encoder instance must not finish a request that has not encoded yet. + + When the encoder cache cannot admit the multi-modal item, the scheduler + schedules only the tokens before it. Finishing that step would hand the + client a successful completion for a request whose image was never + encoded, so no embedding is ever published to the EC connector. + """ + scheduler = create_scheduler( + max_num_seqs=8, + max_num_batched_tokens=1024, + use_ec_connector=True, + ec_role="ec_producer", + ) + request = _make_encoder_instance_request(scheduler) + req_id = request.request_id + + # Encoder budget exhausted: the item cannot be admitted this step. + scheduler.encoder_cache_manager.can_allocate = lambda *a, **k: False + + output = scheduler.schedule() + assert output.num_scheduled_tokens[req_id] > 0 + assert not output.scheduled_encoder_inputs.get(req_id) + assert output.num_scheduled_tokens[req_id] < request.num_prompt_tokens + + scheduler.update_from_output( + output, + make_empty_encoder_model_runner_output(output), + ) + + assert not request.is_finished(), ( + "request finished before its image was encoded, so no embedding was " + "published to the EC connector" + ) + assert request.num_output_tokens == 0 + + # With the budget restored the item is encoded and only then does the + # request finish. + del scheduler.encoder_cache_manager.can_allocate # unshadow the real method + + output = scheduler.schedule() + assert output.scheduled_encoder_inputs.get(req_id) == [0] + + scheduler.update_from_output( + output, + make_empty_encoder_model_runner_output(output), + ) + assert request.is_finished() + + +def test_encoder_instance_finishes_request_once_prompt_is_consumed(): + """The unblocked path completes in a single step, without sampling.""" + scheduler = create_scheduler( + max_num_seqs=8, + max_num_batched_tokens=1024, + use_ec_connector=True, + ec_role="ec_producer", + ) + request = _make_encoder_instance_request(scheduler) + req_id = request.request_id + + output = scheduler.schedule() + assert output.scheduled_encoder_inputs.get(req_id) == [0] + assert output.num_scheduled_tokens[req_id] == request.num_prompt_tokens + + scheduler.update_from_output( + output, + make_empty_encoder_model_runner_output(output), + ) + assert request.status == RequestStatus.FINISHED_STOPPED + # The encoder instance publishes an embedding, not tokens. + assert request.num_output_tokens == 0 diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 99a8b44041..6f1765b179 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -94,6 +94,7 @@ class Scheduler(SchedulerInterface): ) self.structured_output_manager = structured_output_manager self.is_encoder_decoder = vllm_config.model_config.is_encoder_decoder + self.is_encoder_only = vllm_config.is_encoder_only # include_finished_set controls whether a separate set of finished # request ids should be included in the EngineCoreOutputs returned @@ -1813,6 +1814,17 @@ class Scheduler(SchedulerInterface): # Pooling stops as soon as there is output. request.status = RequestStatus.FINISHED_STOPPED stopped = True + elif ( + self.is_encoder_only + and request.num_computed_tokens >= request.num_prompt_tokens + ): + # An encoder instance runs the encoder and publishes the + # embeddings instead of sampling, so it stops as soon as the + # whole prompt is consumed. Encoder inputs are never scheduled + # past a multi-modal item the encoder cache could not admit, so + # a consumed prompt also means every item in it was encoded. + request.status = RequestStatus.FINISHED_STOPPED + stopped = True if new_token_ids and self.structured_output_manager.should_advance( request, new_token_ids=new_token_ids diff --git a/vllm/v1/outputs.py b/vllm/v1/outputs.py index abb6a2be0d..c8876e20b2 100644 --- a/vllm/v1/outputs.py +++ b/vllm/v1/outputs.py @@ -358,8 +358,10 @@ def make_empty_encoder_model_runner_output( # Give every request its own contiguous index req_id_to_index: dict[str, int] = {rid: idx for idx, rid in enumerate(req_ids)} - # No tokens generated yet ⇒ one empty list per request - sampled_token_ids: list[list[int]] = [[0] for _ in req_ids] + # An encoder instance never samples, so it emits no tokens at all. The + # scheduler finishes these requests once their prompt is fully encoded + # (see `Scheduler.update_from_output`). + sampled_token_ids: list[list[int]] = [[] for _ in req_ids] # Pooler outputs are not available yet ⇒ use None placeholders pooler_output: list[torch.Tensor | None] = [None for _ in req_ids]