[Bugfix][EPD][Model Runner V2] Skip gather mm embeddings for encoder only instance (#51222)

Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
Tianyu Guo
2026-08-06 20:40:47 -07:00
committed by GitHub
parent 72c0d67657
commit dd856e48bb
5 changed files with 101 additions and 11 deletions
+31
View File
@@ -5832,3 +5832,34 @@ def test_encoder_instance_finishes_request_once_prompt_is_consumed():
assert request.status == RequestStatus.FINISHED_STOPPED
# The encoder instance publishes an embedding, not tokens.
assert request.num_output_tokens == 0
@pytest.mark.parametrize("ec_role", ["ec_producer", "ec_consumer"])
def test_encoder_input_skipped_when_connector_already_has_the_item(ec_role: str):
"""Neither role re-encodes what the connector already holds.
For a consumer the item is loaded; for a producer there is nothing left to
do at all -- it published that embedding earlier (or a sibling encoder did),
so a second ViT pass would be pure waste. Reached on any repeat: a second
chat turn re-sending its image, a sibling encoder behind the proxy's
round-robin, or a restart that kept the shared storage.
Pinned because the obvious "fix" for the encoder-instance crash this used to
cause is to make the producer encode anyway; the crash belongs to the worker
(an encoder instance must not gather embeddings it never needed), and paying
for it here would cost every deployment a redundant encode.
"""
scheduler = create_scheduler(
max_num_seqs=8,
max_num_batched_tokens=1024,
use_ec_connector=True,
ec_role=ec_role,
)
request = _make_encoder_instance_request(scheduler)
req_id = request.request_id
scheduler.ec_connector.has_cache_item = lambda *a, **k: True
output = scheduler.schedule()
assert output.num_scheduled_tokens[req_id] > 0
assert not output.scheduled_encoder_inputs.get(req_id)