[Bugfix][EPD][Model Runner V2] Skip gather mm embeddings for encoder only instance (#51222)
Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
@@ -138,6 +138,21 @@ class ModelState(ABC):
|
||||
"""Pre-allocated inputs_embeds buffer for dummy runs (contents unused)."""
|
||||
return None
|
||||
|
||||
def execute_mm_encoder(
|
||||
self, scheduled_encoder_inputs: dict[str, list[int]]
|
||||
) -> None:
|
||||
"""Run the multi-modal encoder and cache its outputs by `mm_hash`.
|
||||
|
||||
The encode half of `get_mm_embeddings`, without the gather, for callers
|
||||
that run no language model.
|
||||
"""
|
||||
mm_hashes, mm_kwargs = self.encoder_runner.prepare_mm_inputs(
|
||||
scheduled_encoder_inputs
|
||||
)
|
||||
if mm_kwargs:
|
||||
encoder_outputs = self.encoder_runner.execute_mm_encoder(mm_kwargs)
|
||||
self.encoder_cache.encoder_outputs.update(zip(mm_hashes, encoder_outputs))
|
||||
|
||||
def gather_mm_embeddings(
|
||||
self, input_batch: InputBatch, draft_lookahead: int = 0
|
||||
) -> tuple[list[torch.Tensor], torch.Tensor]:
|
||||
|
||||
Reference in New Issue
Block a user