EVS Support (Video tokens pruning) (#22980)
Signed-off-by: Eugene Khvedchenia <ekhvedchenia@nvidia.com> Signed-off-by: Eugene Khvedchenya <ekhvedchenya@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
co-authored by
Roger Wang
parent
983056e456
commit
392edee34a
@@ -115,6 +115,42 @@ class SupportsMultiModal(Protocol):
|
||||
...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsMultiModalPruning(Protocol):
|
||||
"""The interface required for models that support returning both input
|
||||
embeddings and positions. Model may require custom positions for dynamic
|
||||
pruning of multimodal embeddings.
|
||||
"""
|
||||
supports_multimodal_pruning: ClassVar[Literal[True]] = True
|
||||
|
||||
def recompute_mrope_positions(
|
||||
self, input_ids: list[int],
|
||||
multimodal_embeddings: MultiModalEmbeddings,
|
||||
mrope_positions: torch.LongTensor, num_computed_tokens: int
|
||||
) -> tuple[MultiModalEmbeddings, Tensor, int]:
|
||||
"""
|
||||
Update part of input mrope positions (starting with
|
||||
num_computed_tokens index). Original mrope_positions are computed
|
||||
for unpruned sequence and becomes incorrect once pruning occurs,
|
||||
so once we prune media tokens we should reflect this in the
|
||||
mrope_positions before we feed it to LLM.
|
||||
|
||||
Args:
|
||||
input_ids: (N,) All input tokens of the prompt containing
|
||||
entire sequence.
|
||||
multimodal_embeddings: Tuple of multimodal embeddings that
|
||||
fits into the prefill chunk that is being processed.
|
||||
mrope_positions: Existing mrope positions (3, N) for entire
|
||||
sequence
|
||||
num_computed_tokens: A number of computed tokens so far.
|
||||
|
||||
Returns:
|
||||
Tuple of (multimodal_embeddings, mrope_positions,
|
||||
mrope_position_delta).
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
@overload
|
||||
def supports_multimodal(
|
||||
model: type[object]) -> TypeIs[type[SupportsMultiModal]]:
|
||||
@@ -142,6 +178,25 @@ def supports_multimodal_encoder_tp_data(
|
||||
return getattr(model, "supports_encoder_tp_data", False)
|
||||
|
||||
|
||||
@overload
|
||||
def supports_multimodal_pruning(
|
||||
model: type[object]) -> TypeIs[type[SupportsMultiModalPruning]]:
|
||||
...
|
||||
|
||||
|
||||
@overload
|
||||
def supports_multimodal_pruning(
|
||||
model: object) -> TypeIs[SupportsMultiModalPruning]:
|
||||
...
|
||||
|
||||
|
||||
def supports_multimodal_pruning(
|
||||
model: Union[type[object], object],
|
||||
) -> Union[TypeIs[type[SupportsMultiModalPruning]],
|
||||
TypeIs[SupportsMultiModalPruning]]:
|
||||
return getattr(model, "supports_multimodal_pruning", False)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsScoreTemplate(Protocol):
|
||||
"""The interface required for all models that support score template."""
|
||||
|
||||
Reference in New Issue
Block a user