[Feature] Add VidCom2 video token pruning (#47750)

Signed-off-by: Benedikt Falk <bfalk@nvidia.com>
This commit is contained in:
nvbfalk
2026-07-28 03:20:17 +00:00
committed by GitHub
parent 7aea73d83d
commit 60915c972c
17 changed files with 421 additions and 48 deletions
+8
View File
@@ -41,6 +41,7 @@ if TYPE_CHECKING:
SpeechToTextParams,
VllmConfig,
)
from vllm.config.multimodal import VideoPruningMethod
from vllm.inputs import PromptType, TokensPrompt
from vllm.lora.model_manager import LoRAModelManager
from vllm.model_executor.layers.fused_moe import MoERunner
@@ -424,6 +425,13 @@ class SupportsMultiModalPruning(Protocol):
supports_multimodal_pruning: ClassVar[Literal[True]] = True
supported_video_pruning_methods: ClassVar[tuple["VideoPruningMethod", ...]] = (
"evs",
)
"""Video pruning methods (as reported by
`MultiModalConfig.get_video_pruning_spec`) implemented by this model.
Models supporting methods beyond EVS should override this."""
def recompute_mrope_positions(
self,
input_ids: list[int] | torch.Tensor,
+1 -4
View File
@@ -570,10 +570,7 @@ class InternS1ProForConditionalGeneration(
self.config = config
self.multimodal_config = multimodal_config
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
self.video_pruning_rate = multimodal_config.video_pruning_rate
self.is_multimodal_pruning_enabled = (
multimodal_config.is_multimodal_pruning_enabled()
)
self._init_video_pruning(multimodal_config)
with self._mark_tower_model(vllm_config, {"image", "video"}):
self.visual = Qwen3_VisionTransformer(
@@ -42,10 +42,6 @@ from vllm.model_executor.models.utils import (
maybe_prefix,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.evs import (
compute_retained_tokens_count,
compute_retention_mask,
)
from vllm.multimodal.inputs import (
AudioItem,
BatchedTensorInputs,
@@ -74,6 +70,10 @@ from vllm.multimodal.processing.processor import (
PromptReplacement,
PromptUpdate,
)
from vllm.multimodal.video_prune.evs import (
compute_retained_tokens_count,
compute_retention_mask,
)
from vllm.renderers import TokenizeParams
from vllm.sequence import IntermediateTensors
from vllm.tokenizers import cached_tokenizer_from_config
+6 -6
View File
@@ -67,12 +67,6 @@ from vllm.model_executor.layers.rotary_embedding.common import (
)
from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.evs import (
compute_mrope_for_media,
compute_retained_tokens_count,
compute_retention_mask,
recompute_mrope_positions,
)
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
MultiModalFieldConfig,
@@ -80,6 +74,12 @@ from vllm.multimodal.inputs import (
)
from vllm.multimodal.parse import MultiModalDataItems
from vllm.multimodal.processing import PromptReplacement, PromptUpdate
from vllm.multimodal.video_prune.evs import (
compute_mrope_for_media,
compute_retained_tokens_count,
compute_retention_mask,
recompute_mrope_positions,
)
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape
+61 -24
View File
@@ -50,7 +50,12 @@ from transformers.video_utils import VideoMetadata
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
from vllm.config.multimodal import (
BaseDummyOptions,
MultiModalConfig,
VideoDummyOptions,
VideoPruningMethod,
)
from vllm.distributed import get_pp_group, parallel_state
from vllm.inputs import MultiModalDataDict
from vllm.logger import init_logger
@@ -69,12 +74,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.evs import (
compute_mrope_for_media,
compute_retained_tokens_count,
compute_retention_mask,
recompute_mrope_positions,
)
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
MultiModalFieldConfig,
@@ -92,6 +91,18 @@ from vllm.multimodal.processing import (
PromptUpdate,
PromptUpdateDetails,
)
from vllm.multimodal.video_prune.evs import (
compute_mrope_for_media,
compute_retained_tokens_count,
compute_retention_mask,
recompute_mrope_positions,
)
from vllm.multimodal.video_prune.vidcom2 import (
compute_retained_tokens_count as vidcom2_compute_retained_tokens_count,
)
from vllm.multimodal.video_prune.vidcom2 import (
compute_retention_mask as vidcom2_compute_retention_mask,
)
from vllm.sequence import IntermediateTensors
from vllm.tokenizers.protocol import TokenizerLike
from vllm.tokenizers.registry import cached_tokenizer_from_config
@@ -1256,7 +1267,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
hf_config = self.info.get_hf_config()
tokenizer = self.info.get_tokenizer()
merge_size = hf_config.vision_config.spatial_merge_size
video_pruning_rate = self.info.ctx.get_mm_config().video_pruning_rate
pruning_spec = self.info.ctx.get_mm_config().get_video_pruning_spec()
vision_start_token_id = hf_config.vision_start_token_id
vision_end_token_id = hf_config.vision_end_token_id
video_token_id = hf_config.video_token_id
@@ -1339,11 +1350,18 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
merge_size**2
)
if video_pruning_rate is not None and video_pruning_rate > 0.0:
num_tokens = compute_retained_tokens_count(
# Apply video pruning (EVS or VidCom2) if enabled.
if pruning_spec is not None:
method, prune_q = pruning_spec
count_fn = (
vidcom2_compute_retained_tokens_count
if method == "vidcom2"
else compute_retained_tokens_count
)
num_tokens = count_fn(
tokens_per_frame=tokens_per_frame_base,
num_frames=num_frames,
q=video_pruning_rate,
q=prune_q,
)
tokens_per_frame = [num_tokens] + [0] * (num_frames - 1)
select_token_id = False
@@ -1459,16 +1477,22 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
f"video length ({grid_thw[0]})."
)
# Compute tokens per frame, with EVS support
# Compute tokens per frame, with EVS / VidCom2 support
num_frames = int(grid_thw[0])
tokens_per_frame_base = int(grid_thw[1:].prod()) // merge_length
video_pruning_rate = self.info.ctx.get_mm_config().video_pruning_rate
if video_pruning_rate is not None and video_pruning_rate > 0.0:
num_tokens = compute_retained_tokens_count(
pruning_spec = self.info.ctx.get_mm_config().get_video_pruning_spec()
if pruning_spec is not None:
method, prune_q = pruning_spec
count_fn = (
vidcom2_compute_retained_tokens_count
if method == "vidcom2"
else compute_retained_tokens_count
)
num_tokens = count_fn(
tokens_per_frame=tokens_per_frame_base,
num_frames=num_frames,
q=video_pruning_rate,
q=prune_q,
)
tokens_per_frame = [num_tokens] + [0] * (num_frames - 1)
select_token_id = False
@@ -1701,6 +1725,8 @@ class Qwen3VLForConditionalGeneration(
supports_encoder_tp_data = True
supported_video_pruning_methods = ("evs", "vidcom2")
# To ensure correct weight loading and mapping.
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={
@@ -1719,6 +1745,17 @@ class Qwen3VLForConditionalGeneration(
raise ValueError("Only image or video modality is supported")
def _init_video_pruning(self, multimodal_config: MultiModalConfig) -> None:
pruning_spec = multimodal_config.get_video_pruning_spec()
if pruning_spec is None:
self.video_pruning_method: VideoPruningMethod | None = None
self.video_pruning_rate = multimodal_config.video_pruning_rate
else:
self.video_pruning_method, self.video_pruning_rate = pruning_spec
self.is_multimodal_pruning_enabled = (
multimodal_config.is_multimodal_pruning_enabled()
)
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "model"):
super().__init__()
config: Qwen3VLConfig = vllm_config.model_config.hf_config
@@ -1730,10 +1767,7 @@ class Qwen3VLForConditionalGeneration(
self._tokenizer = cached_tokenizer_from_config(vllm_config.model_config)
self.multimodal_config = multimodal_config
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
self.video_pruning_rate = multimodal_config.video_pruning_rate
self.is_multimodal_pruning_enabled = (
multimodal_config.is_multimodal_pruning_enabled()
)
self._init_video_pruning(multimodal_config)
self.use_deepstack = hasattr(config.vision_config, "deepstack_visual_indexes")
self.deepstack_num_level = (
@@ -1848,7 +1882,7 @@ class Qwen3VLForConditionalGeneration(
EncoderCudaGraphConfig,
)
# When EVS pruning is enabled, embed_multimodal post-processes both
# When video pruning is enabled, embed_multimodal post-processes both
# image and video embeddings (mrope positions are appended for image,
# prune+append for video). The encoder CUDA graph path bypasses that
# post-process, producing inconsistent embedding formats vs eager. So
@@ -2291,9 +2325,12 @@ class Qwen3VLForConditionalGeneration(
t, h, w = size
if self.is_multimodal_pruning_enabled:
# For each video, compute retention mask using EVS.
# retention_mask: [11424].
retention_mask = compute_retention_mask(
# Compute the retention mask for each video (EVS or VidCom2).
if self.video_pruning_method == "vidcom2":
mask_fn = vidcom2_compute_retention_mask
else:
mask_fn = compute_retention_mask
retention_mask = mask_fn(
emb,
size,
spatial_merge_size=self.visual.spatial_merge_size,
+1 -4
View File
@@ -220,10 +220,7 @@ class Qwen3VLMoeForConditionalGeneration(
self._tokenizer = cached_tokenizer_from_config(vllm_config.model_config)
self.multimodal_config = multimodal_config
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
self.video_pruning_rate = multimodal_config.video_pruning_rate
self.is_multimodal_pruning_enabled = (
multimodal_config.is_multimodal_pruning_enabled()
)
self._init_video_pruning(multimodal_config)
self.use_deepstack = hasattr(config.vision_config, "deepstack_visual_indexes")
self.deepstack_num_level = (
+4
View File
@@ -797,6 +797,7 @@ class _ModelInfo:
supports_replayssm: bool
supports_transcription: bool
supports_transcription_only: bool
supported_video_pruning_methods: tuple[str, ...]
@staticmethod
def from_model_cls(model: type[nn.Module]) -> "_ModelInfo":
@@ -827,6 +828,9 @@ class _ModelInfo:
supports_transcription(model) and model.supports_transcription_only
),
has_noops=has_noops(model),
supported_video_pruning_methods=getattr(
model, "supported_video_pruning_methods", ()
),
)