[Bugfix][EPD][Model Runner V2] Skip gather mm embeddings for encoder only instance (#51222)

Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
Tianyu Guo
2026-08-06 20:40:47 -07:00
committed by GitHub
parent 72c0d67657
commit dd856e48bb
5 changed files with 101 additions and 11 deletions
@@ -138,6 +138,21 @@ class ModelState(ABC):
"""Pre-allocated inputs_embeds buffer for dummy runs (contents unused)."""
return None
def execute_mm_encoder(
self, scheduled_encoder_inputs: dict[str, list[int]]
) -> None:
"""Run the multi-modal encoder and cache its outputs by `mm_hash`.
The encode half of `get_mm_embeddings`, without the gather, for callers
that run no language model.
"""
mm_hashes, mm_kwargs = self.encoder_runner.prepare_mm_inputs(
scheduled_encoder_inputs
)
if mm_kwargs:
encoder_outputs = self.encoder_runner.execute_mm_encoder(mm_kwargs)
self.encoder_cache.encoder_outputs.update(zip(mm_hashes, encoder_outputs))
def gather_mm_embeddings(
self, input_batch: InputBatch, draft_lookahead: int = 0
) -> tuple[list[torch.Tensor], torch.Tensor]: