diff --git a/vllm/model_executor/models/qwen2_5_vl.py b/vllm/model_executor/models/qwen2_5_vl.py index e8534c2abb..035317116b 100644 --- a/vllm/model_executor/models/qwen2_5_vl.py +++ b/vllm/model_executor/models/qwen2_5_vl.py @@ -27,7 +27,7 @@ """Inference-only Qwen2.5-VL model compatible with HuggingFace weights.""" from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence -from functools import lru_cache, partial +from functools import partial from typing import Annotated, Any, Literal, TypeAlias import einops @@ -82,6 +82,7 @@ from vllm.multimodal.video_prune.evs import ( ) from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors +from vllm.utils.cache import LRUCache from vllm.utils.tensor_schema import TensorSchema, TensorShape from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d from vllm.v1.attention.backends.registry import AttentionBackendEnum @@ -647,6 +648,7 @@ class Qwen2_5_VisionTransformer(nn.Module): self.spatial_merge_size = vision_config.spatial_merge_size self.fullatt_block_indexes = vision_config.fullatt_block_indexes self.spatial_merge_unit = self.spatial_merge_size**2 + self._rope_by_thw_cache = LRUCache(capacity=1024) use_data_parallel = is_vit_use_data_parallel() self.tp_size = ( 1 @@ -795,8 +797,11 @@ class Qwen2_5_VisionTransformer(nn.Module): return index_new, cu_seqlens_tmp - @lru_cache(maxsize=1024) # noqa: B019 def get_rope_by_thw(self, t, h, w): + cache_key = (t, h, w) + if cache_key in self._rope_by_thw_cache: + return self._rope_by_thw_cache[cache_key] + window_index_thw, cu_seqlens_window_thw = self.get_window_index_thw(t, h, w) cos_thw, sin_thw = self.rotary_pos_emb_thw(t, h, w) @@ -809,13 +814,15 @@ class Qwen2_5_VisionTransformer(nn.Module): cu_seqlens_thw = torch.repeat_interleave( torch.tensor([h * w], dtype=torch.int32), t ) - return ( + result = ( cos_thw, sin_thw, window_index_thw, cu_seqlens_window_thw, cu_seqlens_thw, ) + self._rope_by_thw_cache[cache_key] = result + return result def compute_attn_mask_seqlen( self, diff --git a/vllm/model_executor/models/qwen3_vl.py b/vllm/model_executor/models/qwen3_vl.py index c36d78ce7a..d0fad8fcdb 100644 --- a/vllm/model_executor/models/qwen3_vl.py +++ b/vllm/model_executor/models/qwen3_vl.py @@ -110,6 +110,7 @@ from vllm.sequence import IntermediateTensors from vllm.tokenizers.protocol import TokenizerLike from vllm.tokenizers.registry import cached_tokenizer_from_config from vllm.triton_utils import HAS_TRITON, tl, triton +from vllm.utils.cache import LRUCache from vllm.utils.collection_utils import is_list_of from vllm.utils.math_utils import round_up from vllm.utils.torch_utils import PIN_MEMORY @@ -581,6 +582,7 @@ class Qwen3_VisionTransformer(nn.Module): else [] ) self.num_grid_per_side = int(self.num_position_embeddings**0.5) + self._rot_pos_ids_cache = LRUCache(capacity=1024) use_data_parallel = is_vit_use_data_parallel() self.tp_size = ( @@ -672,9 +674,11 @@ class Qwen3_VisionTransformer(nn.Module): def device(self) -> torch.device: return self.patch_embed.proj.weight.device - @staticmethod - @lru_cache(maxsize=1024) - def rot_pos_ids(h: int, w: int, spatial_merge_size: int) -> torch.Tensor: + def rot_pos_ids(self, h: int, w: int, spatial_merge_size: int) -> torch.Tensor: + cache_key = (h, w, spatial_merge_size) + if cache_key in self._rot_pos_ids_cache: + return self._rot_pos_ids_cache[cache_key] + hpos_ids = np.broadcast_to(np.arange(h).reshape(h, 1), (h, w)) h_div = h // spatial_merge_size w_div = w // spatial_merge_size @@ -697,7 +701,9 @@ class Qwen3_VisionTransformer(nn.Module): wpos_ids = wpos_ids.transpose(0, 2, 1, 3) wpos_ids = wpos_ids.flatten() - return torch.from_numpy(np.stack([hpos_ids, wpos_ids], axis=-1)) + result = torch.from_numpy(np.stack([hpos_ids, wpos_ids], axis=-1)) + self._rot_pos_ids_cache[cache_key] = result + return result def rot_pos_emb(self, grid_thw: list[list[int]]): max_grid_size = max(max(h, w) for _, h, w in grid_thw)