[Bugfix][EPD][Model Runner V2] Skip gather mm embeddings for encoder only instance (#51222)
Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
@@ -1380,9 +1380,17 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
with self.ec_connector.maybe_get_output(
|
||||
scheduler_output
|
||||
) as ec_connector_output:
|
||||
inputs_embeds = self.model_state.get_mm_embeddings(
|
||||
scheduled_encoder_inputs, input_batch, self.req_states
|
||||
)
|
||||
if self.is_encoder_only:
|
||||
# Encode and publish, nothing else: this instance runs no
|
||||
# language model, so the gather inside get_mm_embeddings
|
||||
# would build an inputs_embeds nobody reads -- and it
|
||||
# raises "Encoder cache miss" for any scheduled item this
|
||||
# instance did not encode, taking the engine down with it.
|
||||
self.model_state.execute_mm_encoder(scheduled_encoder_inputs)
|
||||
else:
|
||||
inputs_embeds = self.model_state.get_mm_embeddings(
|
||||
scheduled_encoder_inputs, input_batch, self.req_states
|
||||
)
|
||||
if inputs_embeds is not None and not self.model.requires_raw_input_tokens:
|
||||
input_ids = None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user