[Bugfix][EPD][Model Runner V2] Skip gather mm embeddings for encoder only instance (#51222)
Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
@@ -5832,3 +5832,34 @@ def test_encoder_instance_finishes_request_once_prompt_is_consumed():
|
||||
assert request.status == RequestStatus.FINISHED_STOPPED
|
||||
# The encoder instance publishes an embedding, not tokens.
|
||||
assert request.num_output_tokens == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ec_role", ["ec_producer", "ec_consumer"])
|
||||
def test_encoder_input_skipped_when_connector_already_has_the_item(ec_role: str):
|
||||
"""Neither role re-encodes what the connector already holds.
|
||||
|
||||
For a consumer the item is loaded; for a producer there is nothing left to
|
||||
do at all -- it published that embedding earlier (or a sibling encoder did),
|
||||
so a second ViT pass would be pure waste. Reached on any repeat: a second
|
||||
chat turn re-sending its image, a sibling encoder behind the proxy's
|
||||
round-robin, or a restart that kept the shared storage.
|
||||
|
||||
Pinned because the obvious "fix" for the encoder-instance crash this used to
|
||||
cause is to make the producer encode anyway; the crash belongs to the worker
|
||||
(an encoder instance must not gather embeddings it never needed), and paying
|
||||
for it here would cost every deployment a redundant encode.
|
||||
"""
|
||||
scheduler = create_scheduler(
|
||||
max_num_seqs=8,
|
||||
max_num_batched_tokens=1024,
|
||||
use_ec_connector=True,
|
||||
ec_role=ec_role,
|
||||
)
|
||||
request = _make_encoder_instance_request(scheduler)
|
||||
req_id = request.request_id
|
||||
scheduler.ec_connector.has_cache_item = lambda *a, **k: True
|
||||
|
||||
output = scheduler.schedule()
|
||||
|
||||
assert output.num_scheduled_tokens[req_id] > 0
|
||||
assert not output.scheduled_encoder_inputs.get(req_id)
|
||||
|
||||
Reference in New Issue
Block a user