[Core][Frontend] Add Support for Inference Time mm_processor_kwargs (#9131)
Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com>
This commit is contained in:
@@ -481,6 +481,10 @@ class Sequence:
|
||||
EncoderDecoderLLMInputs,
|
||||
inputs).get("encoder_multi_modal_data")) or {}
|
||||
|
||||
@property
|
||||
def mm_processor_kwargs(self) -> Dict[str, Any]:
|
||||
return self.inputs.get("mm_processor_kwargs") or {}
|
||||
|
||||
@property
|
||||
def lora_int_id(self) -> int:
|
||||
return self.lora_request.lora_int_id if self.lora_request else 0
|
||||
@@ -710,6 +714,14 @@ class SequenceGroup:
|
||||
# We use the multi-modal data of an arbitrary sequence.
|
||||
return self.seqs[0].multi_modal_data
|
||||
|
||||
@property
|
||||
def mm_processor_kwargs(self) -> Dict[str, Any]:
|
||||
# As with multi-modal data, all sequences in the group should have the
|
||||
# same processor kwargs (i.e., mm_processor_kwargs are optionally
|
||||
# provided per request; note that are independent of whether the model
|
||||
# decoder-only or an encoder-decoder).
|
||||
return self.seqs[0].mm_processor_kwargs
|
||||
|
||||
@property
|
||||
def lora_int_id(self) -> int:
|
||||
return self.lora_request.lora_int_id if self.lora_request else 0
|
||||
@@ -949,6 +961,7 @@ class SequenceGroupMetadata(
|
||||
used in prefix caching.
|
||||
state: Internal state tied to this sequence group.
|
||||
multi_modal_data: Multi modal data.
|
||||
mm_processor_kwargs: Multimodal input processor / mapper overrides.
|
||||
encoder_seq_data: Optional sequence data for encoder prompt
|
||||
(SequenceGroup.encoder_seq). Should be None
|
||||
unless you are working with an encoder/decoder
|
||||
@@ -975,6 +988,7 @@ class SequenceGroupMetadata(
|
||||
# "MultiModalDataDict" types. We have to use Any due to msgspec
|
||||
# doesn't allow to have union of 2 different dicts.
|
||||
multi_modal_data: Optional[Any] = None
|
||||
mm_processor_kwargs: Optional[Dict[str, Any]] = None
|
||||
encoder_seq_data: Optional[SequenceData] = None
|
||||
cross_block_table: Optional[List[int]] = None
|
||||
prompt_adapter_request: Optional[PromptAdapterRequest] = None
|
||||
|
||||
Reference in New Issue
Block a user