EVS Support (Video tokens pruning) (#22980)

Signed-off-by: Eugene Khvedchenia <ekhvedchenia@nvidia.com>
Signed-off-by: Eugene Khvedchenya <ekhvedchenya@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
Eugene Khvedchenya
2025-09-26 11:54:54 +08:00
committed by GitHub
co-authored by Roger Wang
parent 983056e456
commit 392edee34a
8 changed files with 783 additions and 39 deletions
+55
View File
@@ -115,6 +115,42 @@ class SupportsMultiModal(Protocol):
...
@runtime_checkable
class SupportsMultiModalPruning(Protocol):
"""The interface required for models that support returning both input
embeddings and positions. Model may require custom positions for dynamic
pruning of multimodal embeddings.
"""
supports_multimodal_pruning: ClassVar[Literal[True]] = True
def recompute_mrope_positions(
self, input_ids: list[int],
multimodal_embeddings: MultiModalEmbeddings,
mrope_positions: torch.LongTensor, num_computed_tokens: int
) -> tuple[MultiModalEmbeddings, Tensor, int]:
"""
Update part of input mrope positions (starting with
num_computed_tokens index). Original mrope_positions are computed
for unpruned sequence and becomes incorrect once pruning occurs,
so once we prune media tokens we should reflect this in the
mrope_positions before we feed it to LLM.
Args:
input_ids: (N,) All input tokens of the prompt containing
entire sequence.
multimodal_embeddings: Tuple of multimodal embeddings that
fits into the prefill chunk that is being processed.
mrope_positions: Existing mrope positions (3, N) for entire
sequence
num_computed_tokens: A number of computed tokens so far.
Returns:
Tuple of (multimodal_embeddings, mrope_positions,
mrope_position_delta).
"""
...
@overload
def supports_multimodal(
model: type[object]) -> TypeIs[type[SupportsMultiModal]]:
@@ -142,6 +178,25 @@ def supports_multimodal_encoder_tp_data(
return getattr(model, "supports_encoder_tp_data", False)
@overload
def supports_multimodal_pruning(
model: type[object]) -> TypeIs[type[SupportsMultiModalPruning]]:
...
@overload
def supports_multimodal_pruning(
model: object) -> TypeIs[SupportsMultiModalPruning]:
...
def supports_multimodal_pruning(
model: Union[type[object], object],
) -> Union[TypeIs[type[SupportsMultiModalPruning]],
TypeIs[SupportsMultiModalPruning]]:
return getattr(model, "supports_multimodal_pruning", False)
@runtime_checkable
class SupportsScoreTemplate(Protocol):
"""The interface required for all models that support score template."""