[Bugfix][EC Connector] Don't stop an encoder-instance request before its images are encoded (#50275)
Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
@@ -108,8 +108,8 @@ async def fanout_encoder_primer(
|
||||
"messages": [
|
||||
{"role": "user", "content": [item]},
|
||||
],
|
||||
# Only need 1 token so the server actually runs the encoder path
|
||||
"max_tokens": 1,
|
||||
# No max_tokens cap: the encoder instance never samples, it finishes
|
||||
# once the prompt is encoded and its embeddings are published.
|
||||
"stream": False,
|
||||
}
|
||||
tasks.append(
|
||||
|
||||
@@ -38,7 +38,12 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
)
|
||||
from vllm.v1.outputs import DraftTokenIds, KVConnectorOutput, ModelRunnerOutput
|
||||
from vllm.v1.outputs import (
|
||||
DraftTokenIds,
|
||||
KVConnectorOutput,
|
||||
ModelRunnerOutput,
|
||||
make_empty_encoder_model_runner_output,
|
||||
)
|
||||
from vllm.v1.request import Request, RequestStatus
|
||||
from vllm.v1.structured_output import StructuredOutputGrammar, StructuredOutputManager
|
||||
|
||||
@@ -5730,3 +5735,100 @@ def test_hybrid_per_group_hit_divergence_fa_deeper_no_external():
|
||||
num_scheduled = output.num_scheduled_tokens[replay.request_id]
|
||||
# Must resume at the convergent boundary (block 0), not the deep FA hit.
|
||||
assert replay.num_tokens - num_scheduled == block_size
|
||||
|
||||
|
||||
def _make_encoder_instance_request(scheduler, text_prefix=8, image_tokens=16):
|
||||
"""One request whose image sits after some text, as an EPD proxy sends it.
|
||||
|
||||
`create_scheduler` builds a text-only model, so `compute_mm_encoder_budget`
|
||||
leaves the encoder cache at zero capacity; give it room so the admitted path
|
||||
is reachable.
|
||||
"""
|
||||
ecm = scheduler.encoder_cache_manager
|
||||
ecm.cache_size = ecm.num_free_slots = ecm.num_freeable_slots = 8 * image_tokens
|
||||
scheduler.max_num_encoder_input_tokens = 8 * image_tokens
|
||||
|
||||
(request,) = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=text_prefix + image_tokens + 4,
|
||||
# Completion must hinge on the prompt being encoded, not on max_tokens.
|
||||
max_tokens=16,
|
||||
mm_hashes_list=[["img-hash-0"]],
|
||||
mm_positions=[[PlaceholderRange(offset=text_prefix, length=image_tokens)]],
|
||||
)
|
||||
scheduler.add_request(request)
|
||||
return request
|
||||
|
||||
|
||||
def test_encoder_instance_defers_stop_until_prompt_is_consumed():
|
||||
"""An encoder instance must not finish a request that has not encoded yet.
|
||||
|
||||
When the encoder cache cannot admit the multi-modal item, the scheduler
|
||||
schedules only the tokens before it. Finishing that step would hand the
|
||||
client a successful completion for a request whose image was never
|
||||
encoded, so no embedding is ever published to the EC connector.
|
||||
"""
|
||||
scheduler = create_scheduler(
|
||||
max_num_seqs=8,
|
||||
max_num_batched_tokens=1024,
|
||||
use_ec_connector=True,
|
||||
ec_role="ec_producer",
|
||||
)
|
||||
request = _make_encoder_instance_request(scheduler)
|
||||
req_id = request.request_id
|
||||
|
||||
# Encoder budget exhausted: the item cannot be admitted this step.
|
||||
scheduler.encoder_cache_manager.can_allocate = lambda *a, **k: False
|
||||
|
||||
output = scheduler.schedule()
|
||||
assert output.num_scheduled_tokens[req_id] > 0
|
||||
assert not output.scheduled_encoder_inputs.get(req_id)
|
||||
assert output.num_scheduled_tokens[req_id] < request.num_prompt_tokens
|
||||
|
||||
scheduler.update_from_output(
|
||||
output,
|
||||
make_empty_encoder_model_runner_output(output),
|
||||
)
|
||||
|
||||
assert not request.is_finished(), (
|
||||
"request finished before its image was encoded, so no embedding was "
|
||||
"published to the EC connector"
|
||||
)
|
||||
assert request.num_output_tokens == 0
|
||||
|
||||
# With the budget restored the item is encoded and only then does the
|
||||
# request finish.
|
||||
del scheduler.encoder_cache_manager.can_allocate # unshadow the real method
|
||||
|
||||
output = scheduler.schedule()
|
||||
assert output.scheduled_encoder_inputs.get(req_id) == [0]
|
||||
|
||||
scheduler.update_from_output(
|
||||
output,
|
||||
make_empty_encoder_model_runner_output(output),
|
||||
)
|
||||
assert request.is_finished()
|
||||
|
||||
|
||||
def test_encoder_instance_finishes_request_once_prompt_is_consumed():
|
||||
"""The unblocked path completes in a single step, without sampling."""
|
||||
scheduler = create_scheduler(
|
||||
max_num_seqs=8,
|
||||
max_num_batched_tokens=1024,
|
||||
use_ec_connector=True,
|
||||
ec_role="ec_producer",
|
||||
)
|
||||
request = _make_encoder_instance_request(scheduler)
|
||||
req_id = request.request_id
|
||||
|
||||
output = scheduler.schedule()
|
||||
assert output.scheduled_encoder_inputs.get(req_id) == [0]
|
||||
assert output.num_scheduled_tokens[req_id] == request.num_prompt_tokens
|
||||
|
||||
scheduler.update_from_output(
|
||||
output,
|
||||
make_empty_encoder_model_runner_output(output),
|
||||
)
|
||||
assert request.status == RequestStatus.FINISHED_STOPPED
|
||||
# The encoder instance publishes an embedding, not tokens.
|
||||
assert request.num_output_tokens == 0
|
||||
|
||||
@@ -94,6 +94,7 @@ class Scheduler(SchedulerInterface):
|
||||
)
|
||||
self.structured_output_manager = structured_output_manager
|
||||
self.is_encoder_decoder = vllm_config.model_config.is_encoder_decoder
|
||||
self.is_encoder_only = vllm_config.is_encoder_only
|
||||
|
||||
# include_finished_set controls whether a separate set of finished
|
||||
# request ids should be included in the EngineCoreOutputs returned
|
||||
@@ -1813,6 +1814,17 @@ class Scheduler(SchedulerInterface):
|
||||
# Pooling stops as soon as there is output.
|
||||
request.status = RequestStatus.FINISHED_STOPPED
|
||||
stopped = True
|
||||
elif (
|
||||
self.is_encoder_only
|
||||
and request.num_computed_tokens >= request.num_prompt_tokens
|
||||
):
|
||||
# An encoder instance runs the encoder and publishes the
|
||||
# embeddings instead of sampling, so it stops as soon as the
|
||||
# whole prompt is consumed. Encoder inputs are never scheduled
|
||||
# past a multi-modal item the encoder cache could not admit, so
|
||||
# a consumed prompt also means every item in it was encoded.
|
||||
request.status = RequestStatus.FINISHED_STOPPED
|
||||
stopped = True
|
||||
|
||||
if new_token_ids and self.structured_output_manager.should_advance(
|
||||
request, new_token_ids=new_token_ids
|
||||
|
||||
+4
-2
@@ -358,8 +358,10 @@ def make_empty_encoder_model_runner_output(
|
||||
# Give every request its own contiguous index
|
||||
req_id_to_index: dict[str, int] = {rid: idx for idx, rid in enumerate(req_ids)}
|
||||
|
||||
# No tokens generated yet ⇒ one empty list per request
|
||||
sampled_token_ids: list[list[int]] = [[0] for _ in req_ids]
|
||||
# An encoder instance never samples, so it emits no tokens at all. The
|
||||
# scheduler finishes these requests once their prompt is fully encoded
|
||||
# (see `Scheduler.update_from_output`).
|
||||
sampled_token_ids: list[list[int]] = [[] for _ in req_ids]
|
||||
|
||||
# Pooler outputs are not available yet ⇒ use None placeholders
|
||||
pooler_output: list[torch.Tensor | None] = [None for _ in req_ids]
|
||||
|
||||
Reference in New Issue
Block a user