[Feature] Add VidCom2 video token pruning (#47750)
Signed-off-by: Benedikt Falk <bfalk@nvidia.com>
This commit is contained in:
@@ -41,6 +41,7 @@ if TYPE_CHECKING:
|
||||
SpeechToTextParams,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.config.multimodal import VideoPruningMethod
|
||||
from vllm.inputs import PromptType, TokensPrompt
|
||||
from vllm.lora.model_manager import LoRAModelManager
|
||||
from vllm.model_executor.layers.fused_moe import MoERunner
|
||||
@@ -424,6 +425,13 @@ class SupportsMultiModalPruning(Protocol):
|
||||
|
||||
supports_multimodal_pruning: ClassVar[Literal[True]] = True
|
||||
|
||||
supported_video_pruning_methods: ClassVar[tuple["VideoPruningMethod", ...]] = (
|
||||
"evs",
|
||||
)
|
||||
"""Video pruning methods (as reported by
|
||||
`MultiModalConfig.get_video_pruning_spec`) implemented by this model.
|
||||
Models supporting methods beyond EVS should override this."""
|
||||
|
||||
def recompute_mrope_positions(
|
||||
self,
|
||||
input_ids: list[int] | torch.Tensor,
|
||||
|
||||
@@ -570,10 +570,7 @@ class InternS1ProForConditionalGeneration(
|
||||
self.config = config
|
||||
self.multimodal_config = multimodal_config
|
||||
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
|
||||
self.video_pruning_rate = multimodal_config.video_pruning_rate
|
||||
self.is_multimodal_pruning_enabled = (
|
||||
multimodal_config.is_multimodal_pruning_enabled()
|
||||
)
|
||||
self._init_video_pruning(multimodal_config)
|
||||
|
||||
with self._mark_tower_model(vllm_config, {"image", "video"}):
|
||||
self.visual = Qwen3_VisionTransformer(
|
||||
|
||||
@@ -42,10 +42,6 @@ from vllm.model_executor.models.utils import (
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.evs import (
|
||||
compute_retained_tokens_count,
|
||||
compute_retention_mask,
|
||||
)
|
||||
from vllm.multimodal.inputs import (
|
||||
AudioItem,
|
||||
BatchedTensorInputs,
|
||||
@@ -74,6 +70,10 @@ from vllm.multimodal.processing.processor import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
)
|
||||
from vllm.multimodal.video_prune.evs import (
|
||||
compute_retained_tokens_count,
|
||||
compute_retention_mask,
|
||||
)
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers import cached_tokenizer_from_config
|
||||
|
||||
@@ -67,12 +67,6 @@ from vllm.model_executor.layers.rotary_embedding.common import (
|
||||
)
|
||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.evs import (
|
||||
compute_mrope_for_media,
|
||||
compute_retained_tokens_count,
|
||||
compute_retention_mask,
|
||||
recompute_mrope_positions,
|
||||
)
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldConfig,
|
||||
@@ -80,6 +74,12 @@ from vllm.multimodal.inputs import (
|
||||
)
|
||||
from vllm.multimodal.parse import MultiModalDataItems
|
||||
from vllm.multimodal.processing import PromptReplacement, PromptUpdate
|
||||
from vllm.multimodal.video_prune.evs import (
|
||||
compute_mrope_for_media,
|
||||
compute_retained_tokens_count,
|
||||
compute_retention_mask,
|
||||
recompute_mrope_positions,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
@@ -50,7 +50,12 @@ from transformers.video_utils import VideoMetadata
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
|
||||
from vllm.config.multimodal import (
|
||||
BaseDummyOptions,
|
||||
MultiModalConfig,
|
||||
VideoDummyOptions,
|
||||
VideoPruningMethod,
|
||||
)
|
||||
from vllm.distributed import get_pp_group, parallel_state
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.logger import init_logger
|
||||
@@ -69,12 +74,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.evs import (
|
||||
compute_mrope_for_media,
|
||||
compute_retained_tokens_count,
|
||||
compute_retention_mask,
|
||||
recompute_mrope_positions,
|
||||
)
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldConfig,
|
||||
@@ -92,6 +91,18 @@ from vllm.multimodal.processing import (
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
)
|
||||
from vllm.multimodal.video_prune.evs import (
|
||||
compute_mrope_for_media,
|
||||
compute_retained_tokens_count,
|
||||
compute_retention_mask,
|
||||
recompute_mrope_positions,
|
||||
)
|
||||
from vllm.multimodal.video_prune.vidcom2 import (
|
||||
compute_retained_tokens_count as vidcom2_compute_retained_tokens_count,
|
||||
)
|
||||
from vllm.multimodal.video_prune.vidcom2 import (
|
||||
compute_retention_mask as vidcom2_compute_retention_mask,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers.protocol import TokenizerLike
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
@@ -1256,7 +1267,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
hf_config = self.info.get_hf_config()
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
merge_size = hf_config.vision_config.spatial_merge_size
|
||||
video_pruning_rate = self.info.ctx.get_mm_config().video_pruning_rate
|
||||
pruning_spec = self.info.ctx.get_mm_config().get_video_pruning_spec()
|
||||
vision_start_token_id = hf_config.vision_start_token_id
|
||||
vision_end_token_id = hf_config.vision_end_token_id
|
||||
video_token_id = hf_config.video_token_id
|
||||
@@ -1339,11 +1350,18 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
merge_size**2
|
||||
)
|
||||
|
||||
if video_pruning_rate is not None and video_pruning_rate > 0.0:
|
||||
num_tokens = compute_retained_tokens_count(
|
||||
# Apply video pruning (EVS or VidCom2) if enabled.
|
||||
if pruning_spec is not None:
|
||||
method, prune_q = pruning_spec
|
||||
count_fn = (
|
||||
vidcom2_compute_retained_tokens_count
|
||||
if method == "vidcom2"
|
||||
else compute_retained_tokens_count
|
||||
)
|
||||
num_tokens = count_fn(
|
||||
tokens_per_frame=tokens_per_frame_base,
|
||||
num_frames=num_frames,
|
||||
q=video_pruning_rate,
|
||||
q=prune_q,
|
||||
)
|
||||
tokens_per_frame = [num_tokens] + [0] * (num_frames - 1)
|
||||
select_token_id = False
|
||||
@@ -1459,16 +1477,22 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
f"video length ({grid_thw[0]})."
|
||||
)
|
||||
|
||||
# Compute tokens per frame, with EVS support
|
||||
# Compute tokens per frame, with EVS / VidCom2 support
|
||||
num_frames = int(grid_thw[0])
|
||||
tokens_per_frame_base = int(grid_thw[1:].prod()) // merge_length
|
||||
|
||||
video_pruning_rate = self.info.ctx.get_mm_config().video_pruning_rate
|
||||
if video_pruning_rate is not None and video_pruning_rate > 0.0:
|
||||
num_tokens = compute_retained_tokens_count(
|
||||
pruning_spec = self.info.ctx.get_mm_config().get_video_pruning_spec()
|
||||
if pruning_spec is not None:
|
||||
method, prune_q = pruning_spec
|
||||
count_fn = (
|
||||
vidcom2_compute_retained_tokens_count
|
||||
if method == "vidcom2"
|
||||
else compute_retained_tokens_count
|
||||
)
|
||||
num_tokens = count_fn(
|
||||
tokens_per_frame=tokens_per_frame_base,
|
||||
num_frames=num_frames,
|
||||
q=video_pruning_rate,
|
||||
q=prune_q,
|
||||
)
|
||||
tokens_per_frame = [num_tokens] + [0] * (num_frames - 1)
|
||||
select_token_id = False
|
||||
@@ -1701,6 +1725,8 @@ class Qwen3VLForConditionalGeneration(
|
||||
|
||||
supports_encoder_tp_data = True
|
||||
|
||||
supported_video_pruning_methods = ("evs", "vidcom2")
|
||||
|
||||
# To ensure correct weight loading and mapping.
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
@@ -1719,6 +1745,17 @@ class Qwen3VLForConditionalGeneration(
|
||||
|
||||
raise ValueError("Only image or video modality is supported")
|
||||
|
||||
def _init_video_pruning(self, multimodal_config: MultiModalConfig) -> None:
|
||||
pruning_spec = multimodal_config.get_video_pruning_spec()
|
||||
if pruning_spec is None:
|
||||
self.video_pruning_method: VideoPruningMethod | None = None
|
||||
self.video_pruning_rate = multimodal_config.video_pruning_rate
|
||||
else:
|
||||
self.video_pruning_method, self.video_pruning_rate = pruning_spec
|
||||
self.is_multimodal_pruning_enabled = (
|
||||
multimodal_config.is_multimodal_pruning_enabled()
|
||||
)
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "model"):
|
||||
super().__init__()
|
||||
config: Qwen3VLConfig = vllm_config.model_config.hf_config
|
||||
@@ -1730,10 +1767,7 @@ class Qwen3VLForConditionalGeneration(
|
||||
self._tokenizer = cached_tokenizer_from_config(vllm_config.model_config)
|
||||
self.multimodal_config = multimodal_config
|
||||
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
|
||||
self.video_pruning_rate = multimodal_config.video_pruning_rate
|
||||
self.is_multimodal_pruning_enabled = (
|
||||
multimodal_config.is_multimodal_pruning_enabled()
|
||||
)
|
||||
self._init_video_pruning(multimodal_config)
|
||||
|
||||
self.use_deepstack = hasattr(config.vision_config, "deepstack_visual_indexes")
|
||||
self.deepstack_num_level = (
|
||||
@@ -1848,7 +1882,7 @@ class Qwen3VLForConditionalGeneration(
|
||||
EncoderCudaGraphConfig,
|
||||
)
|
||||
|
||||
# When EVS pruning is enabled, embed_multimodal post-processes both
|
||||
# When video pruning is enabled, embed_multimodal post-processes both
|
||||
# image and video embeddings (mrope positions are appended for image,
|
||||
# prune+append for video). The encoder CUDA graph path bypasses that
|
||||
# post-process, producing inconsistent embedding formats vs eager. So
|
||||
@@ -2291,9 +2325,12 @@ class Qwen3VLForConditionalGeneration(
|
||||
|
||||
t, h, w = size
|
||||
if self.is_multimodal_pruning_enabled:
|
||||
# For each video, compute retention mask using EVS.
|
||||
# retention_mask: [11424].
|
||||
retention_mask = compute_retention_mask(
|
||||
# Compute the retention mask for each video (EVS or VidCom2).
|
||||
if self.video_pruning_method == "vidcom2":
|
||||
mask_fn = vidcom2_compute_retention_mask
|
||||
else:
|
||||
mask_fn = compute_retention_mask
|
||||
retention_mask = mask_fn(
|
||||
emb,
|
||||
size,
|
||||
spatial_merge_size=self.visual.spatial_merge_size,
|
||||
|
||||
@@ -220,10 +220,7 @@ class Qwen3VLMoeForConditionalGeneration(
|
||||
self._tokenizer = cached_tokenizer_from_config(vllm_config.model_config)
|
||||
self.multimodal_config = multimodal_config
|
||||
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
|
||||
self.video_pruning_rate = multimodal_config.video_pruning_rate
|
||||
self.is_multimodal_pruning_enabled = (
|
||||
multimodal_config.is_multimodal_pruning_enabled()
|
||||
)
|
||||
self._init_video_pruning(multimodal_config)
|
||||
|
||||
self.use_deepstack = hasattr(config.vision_config, "deepstack_visual_indexes")
|
||||
self.deepstack_num_level = (
|
||||
|
||||
@@ -797,6 +797,7 @@ class _ModelInfo:
|
||||
supports_replayssm: bool
|
||||
supports_transcription: bool
|
||||
supports_transcription_only: bool
|
||||
supported_video_pruning_methods: tuple[str, ...]
|
||||
|
||||
@staticmethod
|
||||
def from_model_cls(model: type[nn.Module]) -> "_ModelInfo":
|
||||
@@ -827,6 +828,9 @@ class _ModelInfo:
|
||||
supports_transcription(model) and model.supports_transcription_only
|
||||
),
|
||||
has_noops=has_noops(model),
|
||||
supported_video_pruning_methods=getattr(
|
||||
model, "supported_video_pruning_methods", ()
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user