[V1] VLM preprocessor hashing (#11020)

Signed-off-by: Roger Wang <ywang@roblox.com>
Signed-off-by: Alexander Matveev <alexm@neuralmagic.com>
Co-authored-by: Michael Goin <michael@neuralmagic.com>
Co-authored-by: Roger Wang <ywang@roblox.com>
This commit is contained in:
Alexander Matveev
2024-12-12 00:55:30 +00:00
committed by GitHub
co-authored by Michael Goin Roger Wang
parent 452a723bf2
commit 4e11683368
11 changed files with 332 additions and 48 deletions
+23 -12
View File
@@ -15,7 +15,7 @@ from vllm.sampling_params import SamplingParams
from vllm.transformers_utils.config import try_get_generation_config
from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
from vllm.v1.engine import DetokenizerRequest, EngineCoreRequest
from vllm.v1.engine.mm_input_mapper import MMInputMapper
from vllm.v1.engine.mm_input_mapper import MMHasher, MMInputMapperClient
class Processor:
@@ -42,7 +42,11 @@ class Processor:
model_config)
# Multi-modal (huggingface) input mapper
self.mm_input_mapper = MMInputMapper(model_config)
self.mm_input_mapper_client = MMInputMapperClient(model_config)
# Multi-modal hasher (for images)
self.mm_hasher = MMHasher(
) if model_config.mm_cache_preprocessor else None
# TODO: run in an ThreadpoolExecutor or BackgroundProcess.
# This ideally should releases the GIL, so we should not block the
@@ -71,6 +75,11 @@ class Processor:
assert priority == 0, "vLLM V1 does not support priority at the moment."
assert trace_headers is None, "vLLM V1 does not support tracing yet."
# Compute MM hashes (if enabled)
mm_hashes = None
if self.mm_hasher is not None:
mm_hashes = self.mm_hasher.hash(prompt)
# Process inputs.
preprocessed_inputs = self.input_preprocessor.preprocess(
prompt,
@@ -101,16 +110,17 @@ class Processor:
sampling_params.update_from_generation_config(
self.generation_config_fields, eos_token_id)
# Preprocess multi-modal data
if len(decoder_inputs.multi_modal_data) == 0:
mm_inputs = None
elif isinstance(decoder_inputs.multi_modal_data, MultiModalKwargs):
mm_inputs = [decoder_inputs.multi_modal_data]
else:
mm_inputs = self.mm_input_mapper.process_inputs(
decoder_inputs.multi_modal_data,
decoder_inputs.mm_processor_kwargs,
)
# For merged preprocessor, mm_data is already mm_inputs
precomputed_mm_inputs = None
if isinstance(decoder_inputs.multi_modal_data, MultiModalKwargs):
precomputed_mm_inputs = [decoder_inputs.multi_modal_data]
# Apply MM mapper
mm_inputs = None
if len(decoder_inputs.multi_modal_data) > 0:
mm_inputs, mm_hashes = self.mm_input_mapper_client.process_inputs(
decoder_inputs.multi_modal_data, mm_hashes,
decoder_inputs.mm_processor_kwargs, precomputed_mm_inputs)
# Make Request for Detokenizer.
detokenizer_request = DetokenizerRequest(
@@ -130,6 +140,7 @@ class Processor:
decoder_inputs.prompt,
decoder_inputs.prompt_token_ids,
mm_inputs,
mm_hashes,
decoder_inputs.multi_modal_placeholders,
sampling_params,
eos_token_id,