[Spec Decode] Allow multimodal models with a warning (#41752)
Signed-off-by: Li Zhang <lzhanga@amazon.com> Co-authored-by: Li Zhang <lzhanga@amazon.com>
This commit is contained in:
@@ -80,9 +80,8 @@ class DFlashProposer(SpecDecodeBaseProposer):
|
|||||||
)
|
)
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def _raise_if_multimodal(self):
|
def _warn_if_multimodal(self):
|
||||||
# Override to allow multimodal inputs since DFlash supports Qwen3.5 models
|
# Override to allow multimodal inputs since DFlash supports Qwen3.5 models
|
||||||
# Support for multimodal inputs has not been tested.
|
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@override
|
@override
|
||||||
|
|||||||
@@ -193,7 +193,7 @@ class SpecDecodeBaseProposer:
|
|||||||
|
|
||||||
if self.needs_extra_input_slots:
|
if self.needs_extra_input_slots:
|
||||||
self._raise_if_padded_drafter_batch_disabled()
|
self._raise_if_padded_drafter_batch_disabled()
|
||||||
self._raise_if_multimodal()
|
self._warn_if_multimodal()
|
||||||
self._raise_if_mrope()
|
self._raise_if_mrope()
|
||||||
|
|
||||||
self.is_rejected_token_mask: torch.Tensor | None = None
|
self.is_rejected_token_mask: torch.Tensor | None = None
|
||||||
@@ -309,11 +309,12 @@ class SpecDecodeBaseProposer:
|
|||||||
"disable_padded_drafter_batch in the speculative_config."
|
"disable_padded_drafter_batch in the speculative_config."
|
||||||
)
|
)
|
||||||
|
|
||||||
def _raise_if_multimodal(self):
|
def _warn_if_multimodal(self):
|
||||||
if self.supports_mm_inputs:
|
if self.supports_mm_inputs:
|
||||||
raise NotImplementedError(
|
logger.warning(
|
||||||
"Speculative Decoding with draft models or parallel drafting "
|
"Speculative Decoding with draft models or parallel drafting "
|
||||||
"does not support multimodal models yet"
|
"does not fully support multimodal models yet. "
|
||||||
|
"Proceeding with text-only speculative decoding."
|
||||||
)
|
)
|
||||||
|
|
||||||
def _raise_if_mrope(self):
|
def _raise_if_mrope(self):
|
||||||
|
|||||||
Reference in New Issue
Block a user