[Spec decode] automatically disable mm for text-only draft models (#25667)

Signed-off-by: Jonas Kuebler <kuebj@amazon.com>
This commit is contained in:
Jonas M. Kübler
2025-09-27 08:10:21 +08:00
committed by GitHub
parent 4e33a7ea85
commit 6f5c0931c1
2 changed files with 78 additions and 62 deletions
+14
View File
@@ -804,6 +804,20 @@ class EagleProposer:
self.attn_layer_names = list(draft_attn_layer_names)
if self.is_multimodal_model:
# Even if the target model is multimodal, we can also use
# text-only draft models
try:
dummy_input_ids = torch.tensor([[1]],
device=self.input_ids.device)
self.model.get_input_embeddings(dummy_input_ids,
multimodal_embeddings=None)
except (NotImplementedError, AttributeError, TypeError):
logger.warning(
"Draft model does not support multimodal inputs, "
"falling back to text-only mode")
self.is_multimodal_model = False
if supports_multimodal(target_model):
# handle multimodality
self.model.config.image_token_index = (