[V1] VLM - Run the mm_mapper preprocessor in the frontend process (#10640)

Signed-off-by: Roger Wang <ywang@roblox.com>
Co-authored-by: Michael Goin <michael@neuralmagic.com>
Co-authored-by: Roger Wang <ywang@roblox.com>
This commit is contained in:
Alexander Matveev
2024-12-03 10:33:10 +00:00
committed by GitHub
co-authored by Michael Goin Roger Wang
parent f6084f6324
commit 3bc94cab69
7 changed files with 47 additions and 25 deletions
+11 -2
View File
@@ -14,6 +14,7 @@ from vllm.sampling_params import SamplingParams
from vllm.transformers_utils.config import try_get_generation_config
from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
from vllm.v1.engine import DetokenizerRequest, EngineCoreRequest
from vllm.v1.engine.mm_input_mapper import MMInputMapper
class Processor:
@@ -39,6 +40,9 @@ class Processor:
self.input_processor = input_registry.create_input_processor(
model_config)
# Multi-modal (huggingface) input mapper
self.mm_input_mapper = MMInputMapper(model_config)
# TODO: run in an ThreadpoolExecutor or BackgroundProcess.
# This ideally should releases the GIL, so we should not block the
# asyncio loop while this is running.
@@ -96,6 +100,12 @@ class Processor:
sampling_params.update_from_generation_config(
self.generation_config_fields, eos_token_id)
# Preprocess multi-modal data
mm_inputs = self.mm_input_mapper.process_inputs(
decoder_inputs.multi_modal_data,
decoder_inputs.mm_processor_kwargs) if len(
decoder_inputs.multi_modal_data) > 0 else None
# Make Request for Detokenizer.
detokenizer_request = DetokenizerRequest(
request_id,
@@ -113,9 +123,8 @@ class Processor:
request_id,
decoder_inputs.prompt,
decoder_inputs.prompt_token_ids,
decoder_inputs.multi_modal_data,
mm_inputs,
decoder_inputs.multi_modal_placeholders,
decoder_inputs.mm_processor_kwargs,
sampling_params,
eos_token_id,
arrival_time,