[Core] Use key-only cache for BaseMultiModalProcessor (#23018)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -33,12 +33,13 @@ from vllm.inputs import InputProcessingContext
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.sampling_metadata import SamplingMetadata
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import BaseMultiModalProcessorCache
|
||||
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
|
||||
MultiModalKwargsItems)
|
||||
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
|
||||
from vllm.multimodal.processing import (BaseMultiModalProcessor,
|
||||
BaseProcessingInfo, ProcessingCache,
|
||||
PromptReplacement, PromptUpdate)
|
||||
BaseProcessingInfo, PromptReplacement,
|
||||
PromptUpdate)
|
||||
from vllm.multimodal.profiling import BaseDummyInputsBuilder
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
@@ -367,7 +368,7 @@ def _build_hcxvision_hf_processor(
|
||||
info: HCXVisionProcessingInfo,
|
||||
dummy_inputs: BaseDummyInputsBuilder[HCXVisionProcessingInfo],
|
||||
*,
|
||||
cache: Optional[ProcessingCache] = None,
|
||||
cache: Optional[BaseMultiModalProcessorCache] = None,
|
||||
) -> BaseMultiModalProcessor:
|
||||
if isinstance(info, HCXVisionProcessingInfo):
|
||||
return HCXVisionMultiModalProcessor(
|
||||
|
||||
@@ -22,14 +22,14 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.sampling_metadata import SamplingMetadata
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import BaseMultiModalProcessorCache
|
||||
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
|
||||
MultiModalInputs, MultiModalKwargsItems)
|
||||
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
|
||||
ImageSize, MultiModalDataItems)
|
||||
from vllm.multimodal.processing import (BaseMultiModalProcessor,
|
||||
BaseProcessingInfo, ProcessingCache,
|
||||
PromptReplacement, PromptUpdate,
|
||||
PromptUpdateDetails)
|
||||
BaseProcessingInfo, PromptReplacement,
|
||||
PromptUpdate, PromptUpdateDetails)
|
||||
from vllm.multimodal.profiling import BaseDummyInputsBuilder
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.jsontree import json_map_leaves
|
||||
@@ -394,7 +394,7 @@ def _build_llava_or_pixtral_hf_processor(
|
||||
info: _I,
|
||||
dummy_inputs: BaseDummyInputsBuilder[_I],
|
||||
*,
|
||||
cache: Optional[ProcessingCache] = None,
|
||||
cache: Optional[BaseMultiModalProcessorCache] = None,
|
||||
) -> BaseMultiModalProcessor:
|
||||
if isinstance(info, PixtralHFProcessingInfo):
|
||||
return PixtralHFMultiModalProcessor(
|
||||
|
||||
@@ -58,7 +58,8 @@ from vllm.multimodal.parse import (DictEmbeddingItems, ImageItem,
|
||||
VideoItem, VideoProcessorItems)
|
||||
from vllm.multimodal.processing import (BaseMultiModalProcessor,
|
||||
BaseProcessingInfo, PromptReplacement,
|
||||
PromptUpdate, PromptUpdateDetails)
|
||||
PromptUpdate, PromptUpdateDetails,
|
||||
ResolvedPromptUpdate, _seq2text)
|
||||
from vllm.multimodal.profiling import BaseDummyInputsBuilder
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -744,6 +745,43 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
for modality, pattern in placeholders
|
||||
]
|
||||
|
||||
def _recompute_cached_prompt_update(
|
||||
self,
|
||||
cached_update: ResolvedPromptUpdate,
|
||||
new_item_idx: int,
|
||||
) -> ResolvedPromptUpdate:
|
||||
new_update = super()._recompute_cached_prompt_update(
|
||||
cached_update,
|
||||
new_item_idx,
|
||||
)
|
||||
|
||||
if cached_update.modality == "image":
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
image_processor = self.info.get_image_processor()
|
||||
version = self.info.get_model_version()
|
||||
|
||||
text = _seq2text(tokenizer, cached_update.content.full)
|
||||
prev_item_idx = cached_update.item_idx
|
||||
|
||||
if version == (2, 0) or version == (2, 5):
|
||||
im_start = image_processor.im_start_token
|
||||
im_end = image_processor.im_end_token
|
||||
else:
|
||||
im_start = image_processor.im_id_start
|
||||
im_end = image_processor.im_id_end
|
||||
|
||||
new_update = new_update.with_content(
|
||||
PromptUpdateDetails.select_text(
|
||||
text.replace(
|
||||
f"{im_start}{prev_item_idx}{im_end}",
|
||||
f"{im_start}{new_item_idx}{im_end}",
|
||||
1,
|
||||
),
|
||||
"<unk>",
|
||||
))
|
||||
|
||||
return new_update
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
|
||||
@@ -22,14 +22,14 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
from vllm.model_executor.sampling_metadata import SamplingMetadata
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import BaseMultiModalProcessorCache
|
||||
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
|
||||
MultiModalKwargsItems)
|
||||
from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
|
||||
MultiModalDataItems)
|
||||
from vllm.multimodal.processing import (BaseMultiModalProcessor,
|
||||
BaseProcessingInfo, ProcessingCache,
|
||||
PromptReplacement, PromptUpdate,
|
||||
PromptUpdateDetails)
|
||||
BaseProcessingInfo, PromptReplacement,
|
||||
PromptUpdate, PromptUpdateDetails)
|
||||
from vllm.multimodal.profiling import BaseDummyInputsBuilder
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
@@ -322,7 +322,7 @@ def _build_mistral3_processor(
|
||||
info: _I,
|
||||
dummy_inputs: BaseDummyInputsBuilder[_I],
|
||||
*,
|
||||
cache: Optional[ProcessingCache] = None,
|
||||
cache: Optional[BaseMultiModalProcessorCache] = None,
|
||||
) -> BaseMultiModalProcessor:
|
||||
assert isinstance(info, Mistral3ProcessingInfo)
|
||||
return Mistral3MultiModalProcessor(
|
||||
|
||||
@@ -41,7 +41,8 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
|
||||
BaseProcessingInfo,
|
||||
MultiModalPromptUpdates,
|
||||
PlaceholderFeaturesInfo,
|
||||
PromptReplacement, PromptUpdate)
|
||||
PromptReplacement, PromptUpdate,
|
||||
ResolvedPromptUpdate)
|
||||
# yapf: enable
|
||||
from vllm.multimodal.profiling import BaseDummyInputsBuilder
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -440,6 +441,23 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
|
||||
)
|
||||
]
|
||||
|
||||
def _recompute_cached_prompt_update(
|
||||
self,
|
||||
cached_update: ResolvedPromptUpdate,
|
||||
new_item_idx: int,
|
||||
) -> ResolvedPromptUpdate:
|
||||
new_update = super()._recompute_cached_prompt_update(
|
||||
cached_update,
|
||||
new_item_idx,
|
||||
)
|
||||
|
||||
if cached_update.modality == "image":
|
||||
hf_processor = self.info.get_hf_processor()
|
||||
image_tokens: list[str] = hf_processor.img_tokens # type: ignore
|
||||
new_update = new_update.with_target(image_tokens[new_item_idx])
|
||||
|
||||
return new_update
|
||||
|
||||
def _apply_prompt_updates(
|
||||
self,
|
||||
token_ids: list[int],
|
||||
|
||||
@@ -27,7 +27,7 @@ from vllm.multimodal.parse import (AudioProcessorItems, ImageEmbeddingItems,
|
||||
MultiModalDataItems, MultiModalDataParser)
|
||||
from vllm.multimodal.processing import (BaseMultiModalProcessor,
|
||||
BaseProcessingInfo, PromptReplacement,
|
||||
PromptUpdate)
|
||||
PromptUpdate, ResolvedPromptUpdate)
|
||||
from vllm.multimodal.profiling import BaseDummyInputsBuilder
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils import is_list_of
|
||||
@@ -850,6 +850,25 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
|
||||
),
|
||||
]
|
||||
|
||||
def _recompute_cached_prompt_update(
|
||||
self,
|
||||
cached_update: ResolvedPromptUpdate,
|
||||
new_item_idx: int,
|
||||
) -> ResolvedPromptUpdate:
|
||||
new_update = super()._recompute_cached_prompt_update(
|
||||
cached_update,
|
||||
new_item_idx,
|
||||
)
|
||||
|
||||
if cached_update.modality == "image":
|
||||
image_tokens: list[str] = self.info.image_tokens # type: ignore
|
||||
new_update = new_update.with_target(image_tokens[new_item_idx])
|
||||
elif cached_update.modality == "audio":
|
||||
audio_tokens: list[str] = self.info.audio_tokens # type: ignore
|
||||
new_update = new_update.with_target(audio_tokens[new_item_idx])
|
||||
|
||||
return new_update
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
Phi4MMMultiModalProcessor,
|
||||
|
||||
@@ -25,12 +25,13 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.models.llava import LlavaDummyInputsBuilder
|
||||
from vllm.model_executor.sampling_metadata import SamplingMetadata
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import BaseMultiModalProcessorCache
|
||||
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
|
||||
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
|
||||
ImageSize, MultiModalDataItems)
|
||||
from vllm.multimodal.processing import (BaseMultiModalProcessor,
|
||||
BaseProcessingInfo, ProcessingCache,
|
||||
PromptReplacement, PromptUpdate)
|
||||
BaseProcessingInfo, PromptReplacement,
|
||||
PromptUpdate)
|
||||
from vllm.multimodal.profiling import BaseDummyInputsBuilder
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.jsontree import json_map_leaves
|
||||
@@ -332,7 +333,7 @@ def _build_tarsier_hf_processor(
|
||||
info: _I_Tarsier,
|
||||
dummy_inputs: BaseDummyInputsBuilder[_I_Tarsier],
|
||||
*,
|
||||
cache: Optional[ProcessingCache] = None,
|
||||
cache: Optional[BaseMultiModalProcessorCache] = None,
|
||||
) -> BaseMultiModalProcessor:
|
||||
if isinstance(info, TarsierProcessingInfo):
|
||||
return TarsierMultiModalProcessor(
|
||||
|
||||
Reference in New Issue
Block a user