[Model Runner V2][MM] Support EVS (#46535)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
061043eaca
commit
84c62e1cbd
@@ -49,10 +49,12 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.v1.outputs import LogprobsTensors
|
||||
from vllm.v1.worker.gpu.attn_utils import build_attn_metadata
|
||||
from vllm.v1.worker.gpu.buffer_utils import UvaBackedTensor, async_copy_to_gpu
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
from vllm.v1.worker.gpu.sample.logprob import compute_topk_logprobs
|
||||
from vllm.v1.worker.gpu.sample.output import SamplerOutput
|
||||
from vllm.v1.worker.gpu.sample.penalties import use_penalty
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
|
||||
from .interfaces import (
|
||||
SupportsMultiModal,
|
||||
@@ -869,7 +871,12 @@ class DiffusionGemmaModelState(ModelState):
|
||||
if idx is not None:
|
||||
self.diffusion_states.remove_request(idx)
|
||||
|
||||
def get_mm_embeddings(self, scheduled_encoder_inputs, input_batch):
|
||||
def get_mm_embeddings(
|
||||
self,
|
||||
scheduled_encoder_inputs: dict[str, list[int]],
|
||||
input_batch: InputBatch,
|
||||
req_states: RequestState,
|
||||
) -> torch.Tensor:
|
||||
if not self.supports_mm_inputs:
|
||||
return None
|
||||
|
||||
@@ -880,14 +887,7 @@ class DiffusionGemmaModelState(ModelState):
|
||||
encoder_outputs = self.encoder_runner.execute_mm_encoder(mm_kwargs)
|
||||
self.encoder_cache.encoder_outputs.update(zip(mm_hashes, encoder_outputs))
|
||||
|
||||
mm_embeds, is_mm_embed = self.encoder_runner.gather_mm_embeddings(
|
||||
input_batch.req_ids,
|
||||
input_batch.num_tokens,
|
||||
input_batch.num_scheduled_tokens,
|
||||
input_batch.query_start_loc_np,
|
||||
input_batch.prefill_len_np,
|
||||
input_batch.num_computed_prefill_tokens_np,
|
||||
)
|
||||
mm_embeds, is_mm_embed = self.gather_mm_embeddings(input_batch)
|
||||
|
||||
if not mm_embeds:
|
||||
# No MM tokens in this batch (e.g. all-decode step).
|
||||
|
||||
@@ -421,11 +421,11 @@ class SupportsMultiModalPruning(Protocol):
|
||||
|
||||
def recompute_mrope_positions(
|
||||
self,
|
||||
input_ids: list[int],
|
||||
multimodal_embeddings: MultiModalEmbeddings,
|
||||
input_ids: list[int] | torch.Tensor,
|
||||
multimodal_embeddings: Sequence[torch.Tensor],
|
||||
mrope_positions: torch.LongTensor,
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[MultiModalEmbeddings, Tensor, int]:
|
||||
) -> tuple[Sequence[torch.Tensor], Tensor, int]:
|
||||
"""
|
||||
Update part of input mrope positions (starting with
|
||||
num_computed_tokens index). Original mrope_positions are computed
|
||||
@@ -435,8 +435,9 @@ class SupportsMultiModalPruning(Protocol):
|
||||
|
||||
Args:
|
||||
input_ids: (N,) All input tokens of the prompt containing
|
||||
entire sequence.
|
||||
multimodal_embeddings: Tuple of multimodal embeddings that
|
||||
entire sequence. Either a host-side list or an already
|
||||
device-resident tensor.
|
||||
multimodal_embeddings: Sequence of multimodal embeddings that
|
||||
fits into the prefill chunk that is being processed.
|
||||
mrope_positions: Existing mrope positions (3, N) for entire
|
||||
sequence
|
||||
|
||||
@@ -1583,11 +1583,11 @@ class Qwen2_5_VLForConditionalGeneration(
|
||||
|
||||
def recompute_mrope_positions(
|
||||
self,
|
||||
input_ids: list[int],
|
||||
multimodal_embeddings: tuple[torch.Tensor, ...],
|
||||
input_ids: list[int] | torch.Tensor,
|
||||
multimodal_embeddings: Sequence[torch.Tensor],
|
||||
mrope_positions: torch.LongTensor,
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[tuple[torch.Tensor, ...], torch.Tensor, int]:
|
||||
) -> tuple[Sequence[torch.Tensor], torch.Tensor, int]:
|
||||
"""
|
||||
Update part of input mrope positions (starting with
|
||||
num_computed_tokens index). Original mrope_positions are computed
|
||||
@@ -1618,8 +1618,12 @@ class Qwen2_5_VLForConditionalGeneration(
|
||||
else mrope_positions.device
|
||||
)
|
||||
|
||||
# Tensors.
|
||||
input_ids_t = async_tensor_h2d(input_ids, dtype=torch.long, device=device)
|
||||
# Tensors. input_ids may already be a (device-side) tensor.
|
||||
if isinstance(input_ids, torch.Tensor):
|
||||
assert input_ids.device == device
|
||||
input_ids_t = input_ids.to(torch.long)
|
||||
else:
|
||||
input_ids_t = async_tensor_h2d(input_ids, dtype=torch.long, device=device)
|
||||
|
||||
mm_embeddings_out = [mm[:, :-4] for mm in multimodal_embeddings]
|
||||
mm_embeddings_pos = [
|
||||
@@ -1636,7 +1640,7 @@ class Qwen2_5_VLForConditionalGeneration(
|
||||
video_token_id,
|
||||
)
|
||||
|
||||
return tuple(mm_embeddings_out), positions, mrope_positions_delta
|
||||
return mm_embeddings_out, positions, mrope_positions_delta
|
||||
|
||||
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
|
||||
mm_input_by_modality = {}
|
||||
|
||||
@@ -2657,11 +2657,11 @@ class Qwen3VLForConditionalGeneration(
|
||||
|
||||
def recompute_mrope_positions(
|
||||
self,
|
||||
input_ids: list[int],
|
||||
multimodal_embeddings: MultiModalEmbeddings,
|
||||
input_ids: list[int] | torch.Tensor,
|
||||
multimodal_embeddings: Sequence[torch.Tensor],
|
||||
mrope_positions: torch.LongTensor,
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[MultiModalEmbeddings, torch.Tensor, int]:
|
||||
) -> tuple[Sequence[torch.Tensor], torch.Tensor, int]:
|
||||
"""
|
||||
Update part of input mrope positions (starting with
|
||||
num_computed_tokens index). Original mrope_positions are computed
|
||||
@@ -2672,7 +2672,7 @@ class Qwen3VLForConditionalGeneration(
|
||||
Args:
|
||||
input_ids: (N,) All input tokens of the prompt containing
|
||||
entire sequence.
|
||||
multimodal_embeddings: Tuple of multimodal embeddings that
|
||||
multimodal_embeddings: Sequence of multimodal embeddings that
|
||||
fits into the prefill chunk that is being processed.
|
||||
mrope_positions: Existing mrope positions (3, N) for entire
|
||||
sequence
|
||||
@@ -2694,14 +2694,14 @@ class Qwen3VLForConditionalGeneration(
|
||||
|
||||
@staticmethod
|
||||
def _recompute_mrope_positions(
|
||||
input_ids: list[int],
|
||||
multimodal_embeddings: MultiModalEmbeddings,
|
||||
input_ids: list[int] | torch.Tensor,
|
||||
multimodal_embeddings: Sequence[torch.Tensor],
|
||||
mrope_positions: torch.LongTensor,
|
||||
num_computed_tokens: int,
|
||||
vision_start_token_id: int,
|
||||
image_token_id: int,
|
||||
video_token_id: int,
|
||||
) -> tuple[MultiModalEmbeddings, torch.Tensor, int]:
|
||||
) -> tuple[Sequence[torch.Tensor], torch.Tensor, int]:
|
||||
# Device
|
||||
device = (
|
||||
multimodal_embeddings[0].device
|
||||
@@ -2709,8 +2709,12 @@ class Qwen3VLForConditionalGeneration(
|
||||
else mrope_positions.device
|
||||
)
|
||||
|
||||
# Tensors
|
||||
input_ids_t = async_tensor_h2d(input_ids, device=device, dtype=torch.long)
|
||||
# Tensors. input_ids may already be a (device-side) tensor.
|
||||
if isinstance(input_ids, torch.Tensor):
|
||||
assert input_ids.device == device
|
||||
input_ids_t = input_ids.to(dtype=torch.long)
|
||||
else:
|
||||
input_ids_t = async_tensor_h2d(input_ids, device=device, dtype=torch.long)
|
||||
|
||||
mm_embeddings_out = []
|
||||
mm_embeddings_pos = []
|
||||
@@ -2738,7 +2742,7 @@ class Qwen3VLForConditionalGeneration(
|
||||
video_token_id,
|
||||
)
|
||||
|
||||
return tuple(mm_embeddings_out), positions, mrope_positions_delta
|
||||
return mm_embeddings_out, positions, mrope_positions_delta
|
||||
|
||||
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings | None:
|
||||
mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)
|
||||
|
||||
@@ -90,6 +90,23 @@ class RopeState:
|
||||
def get_positions(self, num_tokens: int) -> torch.Tensor:
|
||||
return self.positions[:, :num_tokens]
|
||||
|
||||
def read_prefill_positions(self, req_idx: int, length: int) -> torch.Tensor:
|
||||
"""Return staged per-request prefill positions as [num_dims, length]."""
|
||||
base = self.num_dims * req_idx
|
||||
return self.prefill_positions.gpu[base : base + self.num_dims, :length]
|
||||
|
||||
def update_prefill_positions(
|
||||
self, req_idx: int, positions: torch.Tensor, delta: int
|
||||
) -> None:
|
||||
"""Overwrite a request's staged prefill positions with recomputed values."""
|
||||
base = self.num_dims * req_idx
|
||||
length = positions.shape[1]
|
||||
self.prefill_positions.gpu[base : base + self.num_dims, :length].copy_(
|
||||
positions
|
||||
)
|
||||
if self.has_delta:
|
||||
self.prefill_delta.np[req_idx] = delta
|
||||
|
||||
def prepare_positions(
|
||||
self,
|
||||
idx_mapping: torch.Tensor,
|
||||
|
||||
@@ -1229,7 +1229,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
scheduled_encoder_inputs=scheduler_output.scheduled_encoder_inputs,
|
||||
)
|
||||
inputs_embeds = self.model_state.get_mm_embeddings(
|
||||
scheduler_output.scheduled_encoder_inputs, input_batch
|
||||
scheduler_output.scheduled_encoder_inputs, input_batch, self.req_states
|
||||
)
|
||||
if inputs_embeds is not None and not self.model.requires_raw_input_tokens:
|
||||
input_ids = None
|
||||
@@ -1413,15 +1413,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# Get cached multimodal embeddings for draft forward.
|
||||
# NOTE: This is done here because postprocess updates
|
||||
# num_computed_prefill_tokens.
|
||||
mm_inputs = self.model_state.encoder_runner.gather_mm_embeddings(
|
||||
input_batch.req_ids,
|
||||
input_batch.num_tokens,
|
||||
input_batch.num_scheduled_tokens,
|
||||
input_batch.query_start_loc_np,
|
||||
input_batch.prefill_len_np,
|
||||
input_batch.num_computed_prefill_tokens_np,
|
||||
# The EAGLE/MTP drafter reads one position ahead of the target.
|
||||
draft_lookahead=1,
|
||||
# The EAGLE/MTP drafter reads one position ahead of the target.
|
||||
mm_inputs = self.model_state.gather_mm_embeddings(
|
||||
input_batch, draft_lookahead=1
|
||||
)
|
||||
|
||||
# Postprocess results and update request states.
|
||||
|
||||
@@ -15,6 +15,7 @@ from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||
from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner
|
||||
from vllm.v1.worker.gpu.mm.rope import get_rope_state
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
from vllm.v1.worker.gpu.model_states.mm_pruning import maybe_create_mm_pruner
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
|
||||
@@ -61,6 +62,11 @@ class DefaultModelState(ModelState):
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
# Pruner is used for multimodal embedding pruning (EVS).
|
||||
self.mm_pruner = maybe_create_mm_pruner(
|
||||
self.model_config, model, self.rope_state, encoder_cache
|
||||
)
|
||||
|
||||
def add_request(self, req_index: int, new_req_data: NewRequestData) -> None:
|
||||
if self.rope_state is not None:
|
||||
assert new_req_data.prefill_token_ids is not None
|
||||
@@ -79,6 +85,7 @@ class DefaultModelState(ModelState):
|
||||
self,
|
||||
scheduled_encoder_inputs: dict[str, list[int]],
|
||||
input_batch: InputBatch,
|
||||
req_states: RequestState,
|
||||
) -> torch.Tensor:
|
||||
mm_hashes, mm_kwargs = self.encoder_runner.prepare_mm_inputs(
|
||||
scheduled_encoder_inputs
|
||||
@@ -89,14 +96,13 @@ class DefaultModelState(ModelState):
|
||||
# Cache the encoder outputs by mm_hash
|
||||
self.encoder_cache.encoder_outputs.update(zip(mm_hashes, encoder_outputs))
|
||||
|
||||
mm_embeds, is_mm_embed = self.encoder_runner.gather_mm_embeddings(
|
||||
input_batch.req_ids,
|
||||
input_batch.num_tokens,
|
||||
input_batch.num_scheduled_tokens,
|
||||
input_batch.query_start_loc_np,
|
||||
input_batch.prefill_len_np,
|
||||
input_batch.num_computed_prefill_tokens_np,
|
||||
)
|
||||
mm_embeds, is_mm_embed = super().gather_mm_embeddings(input_batch)
|
||||
if self.mm_pruner is not None and mm_embeds:
|
||||
# EVS: recompute mrope positions for pruned media.
|
||||
mm_embeds = self.mm_pruner.recompute(mm_embeds, input_batch, req_states)
|
||||
# We must flush the staged rope updates for prepare_inputs() to pick up.
|
||||
self.apply_staged_writes()
|
||||
|
||||
# Use unpadded input_ids to match is_mm_embed size (num_tokens).
|
||||
# input_batch.input_ids may be padded for CUDA graphs.
|
||||
input_ids_unpadded = input_batch.input_ids[: input_batch.num_tokens]
|
||||
@@ -105,6 +111,17 @@ class DefaultModelState(ModelState):
|
||||
)
|
||||
return inputs_embeds[: input_batch.num_tokens_after_padding]
|
||||
|
||||
def gather_mm_embeddings(
|
||||
self, input_batch: InputBatch, draft_lookahead: int = 0
|
||||
) -> tuple[list[torch.Tensor], torch.Tensor]:
|
||||
mm_embeds, is_mm_embed = super().gather_mm_embeddings(
|
||||
input_batch, draft_lookahead
|
||||
)
|
||||
if self.mm_pruner is not None:
|
||||
# EVS: strip the appended mrope-position channels.
|
||||
mm_embeds = self.mm_pruner.strip(mm_embeds)
|
||||
return mm_embeds, is_mm_embed
|
||||
|
||||
def prepare_inputs(
|
||||
self, input_batch: InputBatch, req_states: RequestState
|
||||
) -> dict[str, torch.Tensor | None]:
|
||||
|
||||
@@ -85,7 +85,10 @@ class EncoderDecoderModelState(ModelState):
|
||||
self.encoder_outputs: list[torch.Tensor] = []
|
||||
|
||||
def get_mm_embeddings(
|
||||
self, scheduled_encoder_inputs: dict[str, list[int]], input_batch: InputBatch
|
||||
self,
|
||||
scheduled_encoder_inputs: dict[str, list[int]],
|
||||
input_batch: InputBatch,
|
||||
req_states: RequestState,
|
||||
) -> None:
|
||||
# Ensure encoder inputs are ordered consistently with input_batch.req_ids.
|
||||
encoder_inputs: dict[str, list[int]] = {}
|
||||
|
||||
@@ -13,6 +13,7 @@ from vllm.v1.core.sched.output import NewRequestData
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||
from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
|
||||
@@ -47,6 +48,8 @@ class ModelState(ABC):
|
||||
raise NotImplementedError
|
||||
|
||||
model: nn.Module
|
||||
# Set by mm-capable states; used by the default gather_mm_embeddings().
|
||||
encoder_runner: EncoderRunner
|
||||
|
||||
def get_supported_generation_tasks(self) -> tuple[GenerationTask, ...]:
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
@@ -82,10 +85,27 @@ class ModelState(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def get_mm_embeddings(
|
||||
self, scheduled_encoder_inputs: dict[str, list[int]], input_batch: InputBatch
|
||||
self,
|
||||
scheduled_encoder_inputs: dict[str, list[int]],
|
||||
input_batch: InputBatch,
|
||||
req_states: RequestState,
|
||||
) -> torch.Tensor | None:
|
||||
raise NotImplementedError
|
||||
|
||||
def gather_mm_embeddings(
|
||||
self, input_batch: InputBatch, draft_lookahead: int = 0
|
||||
) -> tuple[list[torch.Tensor], torch.Tensor]:
|
||||
"""Gather cached multimodal embeddings for a speculator's draft forward."""
|
||||
return self.encoder_runner.gather_mm_embeddings(
|
||||
input_batch.req_ids,
|
||||
input_batch.num_tokens,
|
||||
input_batch.num_scheduled_tokens,
|
||||
input_batch.query_start_loc_np,
|
||||
input_batch.prefill_len_np,
|
||||
input_batch.num_computed_prefill_tokens_np,
|
||||
draft_lookahead=draft_lookahead,
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
def prepare_inputs(
|
||||
self, input_batch: InputBatch, req_states: RequestState
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.model_executor.models.interfaces import supports_multimodal_pruning
|
||||
from vllm.multimodal.utils import get_mm_features_in_window
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||
from vllm.v1.worker.gpu.mm.rope import RopeState
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
|
||||
|
||||
class MultiModalPruner:
|
||||
"""Recomputes M-RoPE positions for multimodal models that prune embeddings
|
||||
(e.g. Qwen2.5-VL / Qwen3-VL / Nemotron-Nano-VL Efficient Video Sampling).
|
||||
|
||||
Pruning models append their mrope-position channels to the (variable-count)
|
||||
media embeddings from `embed_multimodal`. Those channels must be split off and
|
||||
used to recompute mrope positions before the embeddings are merged.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: nn.Module,
|
||||
rope_state: RopeState,
|
||||
encoder_cache: EncoderCache,
|
||||
inputs_embeds_size: int,
|
||||
) -> None:
|
||||
self.model = model
|
||||
self.rope_state = rope_state
|
||||
self.encoder_cache = encoder_cache
|
||||
# The cleaned embedding width: pruning models append their mrope-position
|
||||
# channels as trailing columns, so embeds[:, :inputs_embeds_size] strips them.
|
||||
self.inputs_embeds_size = inputs_embeds_size
|
||||
|
||||
def strip(self, mm_embeds: list[torch.Tensor]) -> list[torch.Tensor]:
|
||||
"""Draft forward: strip the appended position channels only.
|
||||
|
||||
Stripping is per-embedding, so no per-request segmentation is needed. The
|
||||
speculator reuses the target's already-recomputed positions, hence there is
|
||||
no position write-back here.
|
||||
"""
|
||||
return [mm[:, : self.inputs_embeds_size] for mm in mm_embeds]
|
||||
|
||||
def recompute(
|
||||
self,
|
||||
mm_embeds: list[torch.Tensor],
|
||||
input_batch: InputBatch,
|
||||
req_states: RequestState,
|
||||
) -> list[torch.Tensor]:
|
||||
"""Target forward: split the appended mrope-position channels off each
|
||||
request's media embeddings, recompute the corrected mrope positions, and
|
||||
stage them back into RopeState. Returns the cleaned, flattened embeddings.
|
||||
"""
|
||||
cleaned: list[torch.Tensor] = []
|
||||
pos = 0
|
||||
req_idx_list = input_batch.idx_mapping_np.tolist()
|
||||
prefill_lens_list = input_batch.prefill_len_np.tolist()
|
||||
num_computed_list = input_batch.num_computed_prefill_tokens_np.tolist()
|
||||
num_scheduled_list = input_batch.num_scheduled_tokens.tolist()
|
||||
for batch_idx, req_id in enumerate(input_batch.req_ids):
|
||||
num_computed = num_computed_list[batch_idx]
|
||||
query_end = num_computed + num_scheduled_list[batch_idx]
|
||||
num_req_embeds = self._num_window_embeds(req_id, num_computed, query_end)
|
||||
if num_req_embeds == 0:
|
||||
continue
|
||||
req_embeds = mm_embeds[pos : pos + num_req_embeds]
|
||||
pos += num_req_embeds
|
||||
|
||||
req_idx = req_idx_list[batch_idx]
|
||||
prefill_len = prefill_lens_list[batch_idx]
|
||||
input_ids = req_states.all_token_ids.gpu[req_idx, :prefill_len]
|
||||
mrope_positions = self.rope_state.read_prefill_positions(
|
||||
req_idx, prefill_len
|
||||
).long()
|
||||
req_cleaned, new_positions, delta = self.model.recompute_mrope_positions(
|
||||
input_ids=input_ids,
|
||||
multimodal_embeddings=req_embeds,
|
||||
mrope_positions=mrope_positions,
|
||||
num_computed_tokens=num_computed,
|
||||
)
|
||||
self.rope_state.update_prefill_positions(req_idx, new_positions, delta)
|
||||
cleaned.extend(req_cleaned)
|
||||
|
||||
assert pos == len(mm_embeds)
|
||||
return cleaned
|
||||
|
||||
def _num_window_embeds(self, req_id: str, query_start: int, query_end: int) -> int:
|
||||
"""Count the media items contributing embeddings to [query_start,
|
||||
query_end), mirroring EncoderRunner.gather_mm_embeddings' per-request
|
||||
windowing so the flat mm_embeds list can be re-segmented per request.
|
||||
|
||||
Note: This logic is intentionally duplicated here rather than being emitted
|
||||
from gather_mm_embeddings, to keep the main path cleaner, since this is a niche
|
||||
feature.
|
||||
"""
|
||||
mm_features = self.encoder_cache.mm_features[req_id]
|
||||
lo, hi = get_mm_features_in_window(
|
||||
mm_features, start=query_start, end=query_end
|
||||
)
|
||||
count = 0
|
||||
for mm_feature in mm_features[lo:hi]:
|
||||
pos_info = mm_feature.mm_position
|
||||
start_idx = max(query_start - pos_info.offset, 0)
|
||||
end_idx = min(query_end - pos_info.offset, pos_info.length)
|
||||
embeds_start, embeds_end = pos_info.get_embeds_indices_in_range(
|
||||
start_idx, end_idx
|
||||
)
|
||||
if embeds_start != embeds_end:
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
def maybe_create_mm_pruner(
|
||||
model_config: ModelConfig,
|
||||
model: nn.Module,
|
||||
rope_state: RopeState | None,
|
||||
encoder_cache: EncoderCache | None,
|
||||
) -> MultiModalPruner | None:
|
||||
"""Create a MultiModalPruner if the model prunes embeddings and uses M-RoPE."""
|
||||
if (
|
||||
not rope_state
|
||||
or not rope_state.has_delta
|
||||
or not encoder_cache
|
||||
or not model_config.multimodal_config
|
||||
or not model_config.multimodal_config.is_multimodal_pruning_enabled()
|
||||
or not supports_multimodal_pruning(model)
|
||||
):
|
||||
return None
|
||||
|
||||
return MultiModalPruner(
|
||||
model, rope_state, encoder_cache, model_config.get_inputs_embeds_size()
|
||||
)
|
||||
Reference in New Issue
Block a user