543 lines
23 KiB
Python
543 lines
23 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
from collections.abc import Mapping
|
|
from pathlib import Path
|
|
from typing import Any, Literal, TypeAlias, TypedDict, cast, final
|
|
|
|
import torch
|
|
from pydantic import ConfigDict, Field, field_validator, model_validator
|
|
from pydantic.dataclasses import dataclass
|
|
|
|
import vllm.envs as envs
|
|
from vllm.config.ec_transfer import ECTransferConfig
|
|
from vllm.config.utils import config, get_from_deprecated_env_if_set
|
|
from vllm.logger import init_logger
|
|
from vllm.utils.hashing import safe_hash
|
|
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
@dataclass
|
|
class BaseDummyOptions:
|
|
"""Base options for generating dummy data during profiling."""
|
|
|
|
count: int = Field(999, ge=0)
|
|
|
|
|
|
@dataclass(config=ConfigDict(extra="forbid"))
|
|
class VideoDummyOptions(BaseDummyOptions):
|
|
"""Options for generating dummy video data during profiling."""
|
|
|
|
num_frames: int | None = Field(None, gt=0)
|
|
width: int | None = Field(None, gt=0)
|
|
height: int | None = Field(None, gt=0)
|
|
|
|
|
|
@dataclass(config=ConfigDict(extra="forbid"))
|
|
class ImageDummyOptions(BaseDummyOptions):
|
|
"""Options for generating dummy image data during profiling."""
|
|
|
|
width: int | None = Field(None, gt=0)
|
|
height: int | None = Field(None, gt=0)
|
|
|
|
|
|
@dataclass(config=ConfigDict(extra="forbid"))
|
|
class AudioDummyOptions(BaseDummyOptions):
|
|
"""Options for generating dummy audio data during profiling."""
|
|
|
|
length: int | None = Field(None, gt=0)
|
|
|
|
|
|
@final
|
|
class MultiModalDummyOptionsBuiltins(TypedDict, total=False):
|
|
"""Type annotations for modality types predefined by vLLM."""
|
|
|
|
image: ImageDummyOptions
|
|
"""Options for dummy images."""
|
|
|
|
video: VideoDummyOptions
|
|
"""Options for dummy videos."""
|
|
|
|
audio: AudioDummyOptions
|
|
"""Options for dummy audios."""
|
|
|
|
|
|
MMEncoderTPMode = Literal["weights", "data"]
|
|
MMCacheType = Literal["shm", "lru"]
|
|
VideoPruningMethod = Literal["evs", "vidcom2"]
|
|
MMTensorIPC = Literal["direct_rpc", "torch_shm"]
|
|
MMHasherAlgorithm = Literal["blake3", "sha256", "sha512"]
|
|
MMProcessorDevice: TypeAlias = str
|
|
"""`"auto"`, `"cpu"`, or the platform's own accelerator name
|
|
(`current_platform.device_type`, e.g. `"cuda"` on CUDA and ROCm,
|
|
`"xpu"` on XPU). Validated against that set by the CLI."""
|
|
|
|
|
|
def _get_mm_hasher_algorithm() -> MMHasherAlgorithm:
|
|
env_value = get_from_deprecated_env_if_set(
|
|
"VLLM_MM_HASHER_ALGORITHM",
|
|
"v0.27",
|
|
"mm_hasher_algorithm",
|
|
)
|
|
env_value = "blake3" if env_value is None else env_value
|
|
return cast(MMHasherAlgorithm, env_value.lower())
|
|
|
|
|
|
MMDummyOptions: TypeAlias = dict[str, BaseDummyOptions]
|
|
"""
|
|
A dictionary containing an entry for each modality type of dummy data.
|
|
|
|
The built-in modalities are defined by
|
|
[`MultiModalDummyOptionsBuiltins`][vllm.config.multimodal.MultiModalDummyOptionsBuiltins].
|
|
"""
|
|
|
|
|
|
@config
|
|
class MultiModalConfig:
|
|
"""Controls the behavior of multimodal models."""
|
|
|
|
language_model_only: bool = False
|
|
"""If True, disables all multimodal inputs by setting all modality limits to 0.
|
|
Equivalent to setting `--limit-mm-per-prompt` to 0 for every modality."""
|
|
limit_per_prompt: MMDummyOptions = Field(default_factory=dict)
|
|
"""The maximum number of input items and options allowed per
|
|
prompt for each modality.
|
|
|
|
Defaults to 999 for each modality.
|
|
|
|
Legacy format (count only):
|
|
{"image": 16, "video": 2}
|
|
|
|
Configurable format (with options):
|
|
{"video": {"count": 1, "num_frames": 32, "width": 512, "height": 512},
|
|
"image": {"count": 5, "width": 512, "height": 512}}
|
|
|
|
Mixed format (combining both):
|
|
{"image": 16, "video": {"count": 1, "num_frames": 32, "width": 512,
|
|
"height": 512}}
|
|
"""
|
|
enable_mm_embeds: bool = False
|
|
"""If `True`, enables passing multimodal embeddings:
|
|
for `LLM` class, this refers to tensor inputs under `multi_modal_data`;
|
|
for the OpenAI-compatible server, this refers to chat messages with content
|
|
`"type": "*_embeds"`.
|
|
|
|
When enabled with `--limit-mm-per-prompt` set to 0 for a modality,
|
|
precomputed embeddings skip count validation for that modality,
|
|
saving memory by not loading encoder modules while still enabling
|
|
embeddings as an input. Limits greater than 0 still apply to embeddings.
|
|
|
|
WARNING: The vLLM engine may crash if incorrect shape of embeddings is passed.
|
|
Only enable this flag for trusted users!"""
|
|
media_io_kwargs: dict[str, dict[str, Any]] = Field(default_factory=dict)
|
|
"""Additional args passed to process media inputs, keyed by modalities.
|
|
For example, to set num_frames for video, set
|
|
`--media-io-kwargs '{"video": {"num_frames": 40} }'`"""
|
|
mm_processor_kwargs: dict[str, object] | None = None
|
|
"""Arguments to be forwarded to the model's processor for multi-modal data,
|
|
e.g., image processor. Overrides for the multi-modal processor obtained
|
|
from `transformers.AutoProcessor.from_pretrained`.
|
|
|
|
The available overrides depend on the model that is being run.
|
|
|
|
For example, for Phi-3-Vision:
|
|
`{"num_crops": 4}`."""
|
|
mm_device_do_normalize: bool | None = True
|
|
"""
|
|
Move the do_normalize computation in the mm preprocessing to before the ViT,
|
|
and let the device do it, so that CPU computation can be saved.
|
|
"""
|
|
mm_processor_cache_gb: float = Field(default=4, ge=0)
|
|
"""The size (in GiB) of the multi-modal processor cache, which is used to
|
|
avoid re-processing past multi-modal inputs.
|
|
|
|
This cache is duplicated for each API process and engine core process,
|
|
resulting in a total memory usage of
|
|
`mm_processor_cache_gb * (api_server_count + data_parallel_size)`.
|
|
|
|
A single processed item larger than this budget is served uncached
|
|
(with a warning) instead of failing. Raise this value to cache such items.
|
|
|
|
Set to `0` to disable this cache completely (not recommended)."""
|
|
mm_processor_cache_type: MMCacheType = "lru"
|
|
"""Type of cache to use for the multi-modal preprocessor/mapper. If `shm`,
|
|
use shared memory FIFO cache. If `lru`, use mirrored LRU cache."""
|
|
mm_hasher_algorithm: MMHasherAlgorithm = Field(
|
|
default_factory=_get_mm_hasher_algorithm
|
|
)
|
|
"""Hash algorithm to use for multi-modal input caching. Use `"sha256"` or
|
|
`"sha512"` for FIPS-compliant deployments."""
|
|
mm_shm_cache_max_object_size_mb: int = Field(default=128, ge=0)
|
|
"""Size limit (in MiB) for each object stored in the multi-modal processor
|
|
shared memory cache. Only effective when `mm_processor_cache_type` is
|
|
`"shm"`."""
|
|
mm_encoder_only: bool = False
|
|
"""
|
|
When enabled, skips the language component of the model.
|
|
|
|
This is usually only valid in disaggregated Encoder process.
|
|
"""
|
|
mm_encoder_tp_mode: MMEncoderTPMode = "weights"
|
|
"""Indicates how to optimize multi-modal encoder inference using tensor
|
|
parallelism (TP).
|
|
|
|
- `"weights"`: Within the same vLLM engine, split the weights of
|
|
each layer across TP ranks. (default TP behavior)
|
|
- `"data"`: Within the same vLLM engine, split the batched input data
|
|
across TP ranks to process the data in parallel, while hosting
|
|
the full weights on each TP rank.
|
|
This batch-level DP is not to be confused with API request-level
|
|
DP (which is controlled by `--data-parallel-size`).
|
|
This is only supported on a per-model basis and falls back to
|
|
`"weights"` if the encoder does not support DP."""
|
|
mm_encoder_attn_backend: AttentionBackendEnum | None = None
|
|
"""Optional override for the multi-modal encoder attention backend when
|
|
using vision transformers. Accepts any value from
|
|
`vllm.v1.attention.backends.registry.AttentionBackendEnum` (e.g. `FLASH_ATTN`)."""
|
|
mm_encoder_attn_dtype: Literal["fp8"] | None = None
|
|
"""Optional dtype override for ViT encoder attention. Set to `"fp8"` to
|
|
enable FP8 quantization via the FlashInfer cuDNN backend. When set to
|
|
`"fp8"` without a scale file, dynamic scaling is used automatically.
|
|
See docs/features/quantization/fp8_vit_attn.md for details."""
|
|
mm_encoder_fp8_scale_path: str | None = None
|
|
"""Path to a JSON file containing per-layer FP8 Q/K/V scales for ViT
|
|
encoder attention. When provided (with `mm_encoder_attn_dtype="fp8"`),
|
|
static scaling is used. When omitted, dynamic scaling is used."""
|
|
mm_encoder_fp8_scale_save_path: str | None = None
|
|
"""When set with dynamic FP8 scaling (`mm_encoder_attn_dtype="fp8"`
|
|
and no `mm_encoder_fp8_scale_path`), saves the calibrated scales to
|
|
this file after the amax history buffer is full. The saved file can
|
|
then be used as `mm_encoder_fp8_scale_path` in subsequent runs."""
|
|
mm_encoder_fp8_scale_save_margin: float = Field(default=1.5, gt=0.0)
|
|
"""Safety margin multiplied onto scales when auto-saving. A value > 1
|
|
leaves headroom so that inputs with larger activations than the
|
|
calibration set do not overflow FP8 range. Default 1.5."""
|
|
interleave_mm_strings: bool = False
|
|
"""Enable fully interleaved support for multimodal prompts, while using
|
|
--chat-template-content-format=string."""
|
|
skip_mm_profiling: bool = False
|
|
"""When enabled, skips multimodal memory profiling and only profiles with
|
|
language backbone model during engine initialization.
|
|
|
|
This reduces engine startup time but shifts the responsibility to users for
|
|
estimating the peak memory usage of the activation of multimodal encoder and
|
|
embedding cache."""
|
|
video_pruning_rate: float | None = Field(default=None, ge=0.0, lt=1.0)
|
|
"""Fraction of video tokens to prune from each video. Value sits in range
|
|
[0;1); pruning is enabled when it is greater than 0. The pruning algorithm
|
|
is selected by `video_pruning_method`.
|
|
"""
|
|
video_pruning_method: VideoPruningMethod = "evs"
|
|
"""Video token pruning algorithm applied when `video_pruning_rate` > 0:
|
|
- "evs": Efficient Video Sampling.
|
|
- "vidcom2": Video Compression Commander.
|
|
"""
|
|
mm_tensor_ipc: MMTensorIPC = "direct_rpc"
|
|
"""IPC (inter-process communication) method for multimodal tensors.
|
|
- "direct_rpc": Use msgspec serialization via RPC
|
|
- "torch_shm": Use torch.multiprocessing shared memory for zero-copy IPC
|
|
Defaults to "direct_rpc". """
|
|
allow_missing_mm_embeddings: bool = False
|
|
"""Whether a pre-computed-embedding input may omit the `*_embeds` tensor.
|
|
|
|
In an encode/prefill/decode (EPD) deployment the encoder instance publishes
|
|
embeddings through the EC connector. An EC consumer loads those embeddings
|
|
from the connector, while a KV consumer receives the resulting prompt KV
|
|
cache. Their requests only need the grid/size metadata that sizes the
|
|
placeholder range.
|
|
|
|
Derived, not user-settable: `VllmConfig.__post_init__` sets this to True
|
|
on EC and KV consumers. Everywhere else it stays False so that a request
|
|
which forgets its embeddings still fails fast in the frontend, with a clear
|
|
error, rather than deep inside the model."""
|
|
|
|
mm_ipc_gpu_memory_gb: float = Field(default=0, ge=0)
|
|
"""Amount of GPU memory (in GiB) sequestered on the engine's device for
|
|
GPU-side multimodal work in the API-server (frontend) process, such as
|
|
hardware video decoding.
|
|
|
|
This budget is carved out of the engine's KV-cache memory so the headroom
|
|
physically exists, and frontend GPU decode paths acquire from a blocking
|
|
byte-counting semaphore of this size before allocating on the device.
|
|
|
|
Set to `0` (default) to disable frontend GPU multimodal memory gating."""
|
|
|
|
@field_validator("limit_per_prompt", mode="before")
|
|
@classmethod
|
|
def _validate_limit_per_prompt(
|
|
cls,
|
|
value: dict[str, int | dict[str, int]],
|
|
) -> MMDummyOptions:
|
|
out: MMDummyOptions = {}
|
|
|
|
for k, v in value.items():
|
|
# Handle legacy format where only count is specified
|
|
if isinstance(v, int):
|
|
v = {"count": v}
|
|
|
|
# Convert to the appropriate DummyOptions subclass
|
|
if k == "video":
|
|
out[k] = VideoDummyOptions(**v)
|
|
elif k == "image":
|
|
out[k] = ImageDummyOptions(**v)
|
|
elif k == "audio":
|
|
out[k] = AudioDummyOptions(**v)
|
|
else:
|
|
out[k] = BaseDummyOptions(**v)
|
|
|
|
return out
|
|
|
|
@field_validator("mm_encoder_attn_backend", mode="before")
|
|
@classmethod
|
|
def _validate_mm_encoder_attn_backend(
|
|
cls, value: str | AttentionBackendEnum | None
|
|
) -> AttentionBackendEnum | None:
|
|
if isinstance(value, str) and value.upper() == "XFORMERS":
|
|
raise ValueError(
|
|
"Attention backend 'XFORMERS' has been removed (See PR #29262 for "
|
|
"details). Please select a supported attention backend."
|
|
)
|
|
|
|
if value is None or isinstance(value, AttentionBackendEnum):
|
|
return value
|
|
|
|
assert isinstance(value, str), (
|
|
"mm_encoder_attn_backend must be a string or an AttentionBackendEnum."
|
|
)
|
|
return AttentionBackendEnum[value.upper()]
|
|
|
|
@model_validator(mode="after")
|
|
def _validate_multimodal_config(self):
|
|
if self.mm_processor_cache_type != "shm" and (
|
|
self.mm_shm_cache_max_object_size_mb
|
|
!= MultiModalConfig.mm_shm_cache_max_object_size_mb
|
|
):
|
|
raise ValueError(
|
|
"'mm_shm_cache_max_object_size_mb' should only be set when "
|
|
"'mm_processor_cache_type' is 'shm'."
|
|
)
|
|
# Validate FP8 scale path combinations.
|
|
if self.mm_encoder_attn_dtype != "fp8" and (
|
|
self.mm_encoder_fp8_scale_path is not None
|
|
or self.mm_encoder_fp8_scale_save_path is not None
|
|
):
|
|
raise ValueError(
|
|
"'mm_encoder_fp8_scale_path' and "
|
|
"'mm_encoder_fp8_scale_save_path' require "
|
|
"'mm_encoder_attn_dtype' to be 'fp8'."
|
|
)
|
|
if (
|
|
self.mm_encoder_fp8_scale_path is not None
|
|
and self.mm_encoder_fp8_scale_save_path is not None
|
|
):
|
|
raise ValueError(
|
|
"'mm_encoder_fp8_scale_save_path' cannot be used with "
|
|
"'mm_encoder_fp8_scale_path' (saving requires dynamic scaling)."
|
|
)
|
|
|
|
# Validate file paths exist.
|
|
if self.mm_encoder_fp8_scale_path is not None:
|
|
scale_path = Path(self.mm_encoder_fp8_scale_path)
|
|
if not scale_path.is_file():
|
|
raise FileNotFoundError(f"FP8 scale file not found: {scale_path}")
|
|
if self.mm_encoder_fp8_scale_save_path is not None:
|
|
save_parent = Path(self.mm_encoder_fp8_scale_save_path).parent
|
|
if not save_parent.is_dir():
|
|
raise FileNotFoundError(
|
|
f"Parent directory for FP8 scale save path not found: {save_parent}"
|
|
)
|
|
return self
|
|
|
|
@staticmethod
|
|
def fold_mm_processor_device(
|
|
mm_processor_kwargs: dict[str, Any] | None,
|
|
mm_processor_device: MMProcessorDevice | None,
|
|
) -> dict[str, Any] | None:
|
|
"""Fold the `mm_processor_device` convenience flag into the kwargs.
|
|
|
|
The flag keeps no state of its own: `mm_processor_kwargs["device"]` is
|
|
the only representation of where the processor runs, so an explicit
|
|
`device` there always wins and `"auto"` stays unresolved for
|
|
`VllmConfig`, which is where the EC role needed to resolve it lives.
|
|
|
|
Args:
|
|
mm_processor_kwargs: The kwargs as given, or None.
|
|
mm_processor_device: The flag's value, or None when unset.
|
|
|
|
Returns:
|
|
The kwargs to build the config with, unchanged unless the flag adds
|
|
a `device`.
|
|
"""
|
|
if mm_processor_device in (None, "auto"):
|
|
return mm_processor_kwargs
|
|
if (mm_processor_kwargs or {}).get("device") is not None:
|
|
return mm_processor_kwargs
|
|
|
|
from vllm.platforms import current_platform
|
|
|
|
# Any explicit value other than "cpu" means "the accelerator", so a
|
|
# programmatically-set "cuda" still works on a platform whose device type
|
|
# is named differently ("xpu"), and degrades to CPU where there is none.
|
|
device = (
|
|
"cpu"
|
|
if mm_processor_device == "cpu"
|
|
else (current_platform.device_type or "cpu")
|
|
)
|
|
return {**(mm_processor_kwargs or {}), "device": device}
|
|
|
|
def get_mm_processor_device_type(self) -> str | None:
|
|
"""The torch device type `mm_processor_kwargs["device"]` names.
|
|
|
|
`mm_processor_kwargs` is untyped, so `device` may be any form torch
|
|
accepts -- `"cuda"`, `"cuda:1"`, `torch.device(...)`, or a bare index.
|
|
Normalising through torch rather than parsing the string keeps the
|
|
non-string forms from slipping past a caller's comparison.
|
|
|
|
Returns:
|
|
The device type, or None when no device is requested.
|
|
|
|
Raises:
|
|
ValueError: If `device` is not something `torch.device` accepts.
|
|
`validate_mm_processor_device` is what surfaces this during
|
|
startup, so the value is only parsed once.
|
|
"""
|
|
device = (self.mm_processor_kwargs or {}).get("device")
|
|
if device is None:
|
|
return None
|
|
try:
|
|
return torch.device(device).type # type: ignore[arg-type]
|
|
except (RuntimeError, TypeError, ValueError):
|
|
raise ValueError(
|
|
f'Invalid "device" in mm_processor_kwargs: {device!r}. Expected a '
|
|
'torch device such as "cpu", "cuda" or "cuda:0".'
|
|
) from None
|
|
|
|
def validate_mm_processor_device(self, ec_config: ECTransferConfig | None) -> None:
|
|
"""Check `mm_processor_kwargs["device"]` for this deployment.
|
|
|
|
The only place the requested device is validated, so it runs even on a
|
|
CPU-only platform: the value is parsed before any early return.
|
|
|
|
Args:
|
|
ec_config: The deployment's EC config, or None when it is not an
|
|
encode/prefill/decode deployment. Passed in because it is not
|
|
reachable from here, and because a field assigned after
|
|
construction would not re-trigger this config's validators.
|
|
|
|
Raises:
|
|
ValueError: If the requested device is not a torch device, or if it
|
|
is the accelerator on an instance that also runs the language
|
|
model.
|
|
"""
|
|
from vllm.platforms import current_platform
|
|
|
|
device_type = self.get_mm_processor_device_type()
|
|
accelerator = current_platform.device_type
|
|
if device_type is None or accelerator in ("", "cpu"):
|
|
return
|
|
if device_type != accelerator:
|
|
return
|
|
|
|
if ec_config is None or not ec_config.is_encode_only:
|
|
raise ValueError(
|
|
f"Cannot run the multi-modal processor on {device_type!r}: this "
|
|
"instance also runs the language model. The processor would "
|
|
"share the device with the model's forward pass, so its "
|
|
"transform kernels contend with that compute, and because it "
|
|
"runs in the API-server process its allocations are outside the "
|
|
"memory the engine profiled for its KV cache -- risking OOM or "
|
|
"a silently shrunken cache.\n"
|
|
"Accelerator preprocessing is only supported on an encode-only "
|
|
"instance of an encode/prefill/decode deployment (an EC "
|
|
"producer that is not also a consumer), which runs no forward "
|
|
"pass and allocates no KV cache.\n"
|
|
'Use --mm-processor-device=cpu, or drop "device" from '
|
|
"--mm-processor-kwargs."
|
|
)
|
|
|
|
logger.info_once(
|
|
"Running the multi-modal processor on %s. Override with "
|
|
"--mm-processor-device=cpu.",
|
|
device_type,
|
|
)
|
|
|
|
def compute_hash(self) -> str:
|
|
"""
|
|
WARNING: Whenever a new field is added to this config,
|
|
ensure that it is included in the factors list if
|
|
it affects the computation graph.
|
|
|
|
Provide a hash that uniquely identifies all the configs
|
|
that affect the structure of the computation
|
|
graph from input ids/embeddings to the final hidden states,
|
|
excluding anything before input ids/embeddings and after
|
|
the final hidden states.
|
|
"""
|
|
factors: list[Any] = [
|
|
self.mm_encoder_attn_backend.name
|
|
if self.mm_encoder_attn_backend is not None
|
|
else None,
|
|
self.mm_encoder_tp_mode,
|
|
self.mm_encoder_attn_dtype,
|
|
self.mm_encoder_fp8_scale_path,
|
|
self.mm_device_do_normalize,
|
|
]
|
|
hash_str = safe_hash(str(factors).encode(), usedforsecurity=False).hexdigest()
|
|
return hash_str
|
|
|
|
def get_limit_per_prompt(self, modality: str) -> int:
|
|
"""
|
|
Get the maximum number of input items allowed per prompt
|
|
for the given modality (backward compatible).
|
|
"""
|
|
if self.language_model_only:
|
|
return 0
|
|
|
|
limit_data = self.limit_per_prompt.get(modality)
|
|
|
|
if limit_data is None:
|
|
# Unspecified modality is set to 999 by default
|
|
return 999
|
|
|
|
return limit_data.count
|
|
|
|
def merge_mm_processor_kwargs(
|
|
self,
|
|
inference_kwargs: Mapping[str, object],
|
|
) -> dict[str, object]:
|
|
"""
|
|
Get the keyword arguments to pass to the multi-modal processor
|
|
according to the extra arguments passed during inference.
|
|
"""
|
|
kwargs = self.mm_processor_kwargs or {}
|
|
if self.mm_device_do_normalize:
|
|
kwargs["do_normalize"] = False
|
|
kwargs["do_rescale"] = False
|
|
return kwargs | dict(inference_kwargs)
|
|
|
|
def use_gpu_video_backend(self) -> bool:
|
|
"""Return whether the configured video loader or codec uses the GPU."""
|
|
from vllm.multimodal.video import VIDEO_LOADER_REGISTRY
|
|
|
|
video_kwargs = self.media_io_kwargs.get("video", {})
|
|
video_loader_backend = (
|
|
video_kwargs.get("video_backend") or envs.VLLM_VIDEO_LOADER_BACKEND
|
|
)
|
|
codec_backend = video_kwargs.get("backend")
|
|
return VIDEO_LOADER_REGISTRY.backend_requires_gpu(video_loader_backend) or (
|
|
codec_backend is not None
|
|
and VIDEO_LOADER_REGISTRY.backend_requires_gpu(codec_backend)
|
|
)
|
|
|
|
def is_multimodal_pruning_enabled(self):
|
|
return self.get_video_pruning_spec() is not None
|
|
|
|
def get_video_pruning_spec(self) -> tuple[VideoPruningMethod, float] | None:
|
|
"""Return `(method, rate)` when video pruning is enabled, else None.
|
|
`rate` is the fraction of video tokens to prune."""
|
|
if self.video_pruning_rate is not None and self.video_pruning_rate > 0:
|
|
return (self.video_pruning_method, float(self.video_pruning_rate))
|
|
return None
|