[Refactor] Decouple TimingContext from InputProcessingContext (#35083)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -1,7 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from .context import BaseProcessingInfo, InputProcessingContext
|
||||
from .dummy_inputs import BaseDummyInputsBuilder, ProcessorInputs
|
||||
from .context import BaseProcessingInfo, InputProcessingContext, TimingContext
|
||||
from .dummy_inputs import BaseDummyInputsBuilder
|
||||
from .inputs import ProcessorInputs
|
||||
from .processor import (
|
||||
BaseMultiModalProcessor,
|
||||
EncDecMultiModalProcessor,
|
||||
@@ -15,6 +16,7 @@ from .processor import (
|
||||
__all__ = [
|
||||
"BaseProcessingInfo",
|
||||
"InputProcessingContext",
|
||||
"TimingContext",
|
||||
"BaseDummyInputsBuilder",
|
||||
"ProcessorInputs",
|
||||
"BaseMultiModalProcessor",
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import contextvars
|
||||
import threading
|
||||
import time
|
||||
from abc import abstractmethod
|
||||
from collections.abc import Generator, Mapping
|
||||
from collections.abc import Mapping
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from functools import cached_property
|
||||
@@ -33,104 +31,53 @@ if TYPE_CHECKING:
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
from transformers.processing_utils import ProcessorMixin
|
||||
|
||||
from vllm.config import ModelConfig, ObservabilityConfig
|
||||
from vllm.config import ModelConfig
|
||||
else:
|
||||
PretrainedConfig = object
|
||||
BatchFeature = object
|
||||
ProcessorMixin = object
|
||||
|
||||
ModelConfig = object
|
||||
ObservabilityConfig = object
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
_request_id_context: contextvars.ContextVar[str | None] = contextvars.ContextVar(
|
||||
"_request_id_context", default=None
|
||||
)
|
||||
|
||||
|
||||
def get_current_request_id() -> str | None:
|
||||
"""Get the current request_id from the context, if available."""
|
||||
return _request_id_context.get()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def set_request_id(request_id: str) -> Generator[None, None, None]:
|
||||
"""Context manager to set the request_id for the current context."""
|
||||
token = _request_id_context.set(request_id)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_request_id_context.reset(token)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MultiModalProcessorTimingStats:
|
||||
"""Per-request timing statistics for multimodal processor stages."""
|
||||
class TimingContext:
|
||||
"""Helper class to record execution times during multi-modal processing."""
|
||||
|
||||
hf_processor_time: float = 0.0
|
||||
"""Time spent in HuggingFace processor calls (seconds)."""
|
||||
enabled: bool = True
|
||||
"""If disabled, `TimingContext.record` becomes a no-op."""
|
||||
|
||||
hashing_time: float = 0.0
|
||||
"""Time spent computing multimodal item hashes (seconds)."""
|
||||
stage_secs: dict[str, float] = field(default_factory=dict)
|
||||
"""The execution time (in seconds) for each processing stage."""
|
||||
|
||||
cache_lookup_time: float = 0.0
|
||||
"""Time spent in cache lookups and merges (seconds)."""
|
||||
@property
|
||||
def total_secs(self) -> float:
|
||||
return sum(self.stage_secs.values())
|
||||
|
||||
prompt_update_time: float = 0.0
|
||||
"""Time spent applying prompt updates and finding placeholders (seconds)."""
|
||||
@contextmanager
|
||||
def record(self, stage: str):
|
||||
"""Record the execution time for a processing stage."""
|
||||
if not self.enabled:
|
||||
yield
|
||||
return
|
||||
|
||||
preprocessor_total_time: float = 0.0
|
||||
"""Total preprocessing time (seconds)."""
|
||||
start_time = time.perf_counter()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
elapsed = time.perf_counter() - start_time
|
||||
self.stage_secs.setdefault(stage, 0.0)
|
||||
self.stage_secs[stage] += elapsed
|
||||
|
||||
def to_dict(self) -> dict[str, float]:
|
||||
"""Convert stats to a dictionary for JSON serialization."""
|
||||
return {
|
||||
"hf_processor_time": self.hf_processor_time,
|
||||
"hashing_time": self.hashing_time,
|
||||
"cache_lookup_time": self.cache_lookup_time,
|
||||
"prompt_update_time": self.prompt_update_time,
|
||||
"preprocessor_total_time": self.preprocessor_total_time,
|
||||
def get_stats_dict(self):
|
||||
stats_dict = {
|
||||
f"{stage}_secs": time_s for stage, time_s in self.stage_secs.items()
|
||||
}
|
||||
stats_dict["preprocessor_total_secs"] = self.total_secs
|
||||
|
||||
|
||||
@contextmanager
|
||||
def timed_preprocessor_operation(ctx: "InputProcessingContext", stage_name: str):
|
||||
"""
|
||||
Context manager to time an operation using the context's timing stats.
|
||||
|
||||
The request_id is automatically retrieved from the context variable,
|
||||
so it doesn't need to be passed as a parameter.
|
||||
|
||||
Args:
|
||||
ctx: The InputProcessingContext containing the timing stats registry.
|
||||
stage_name: Name of the stage being timed.
|
||||
"""
|
||||
request_id = get_current_request_id()
|
||||
if ctx is None or request_id is None:
|
||||
yield
|
||||
return
|
||||
|
||||
stats = ctx.get_timing_stats(request_id)
|
||||
if stats is None:
|
||||
yield
|
||||
return
|
||||
|
||||
start_time = time.perf_counter()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
elapsed = time.perf_counter() - start_time
|
||||
if stage_name == "hf_processor":
|
||||
stats.hf_processor_time += elapsed
|
||||
elif stage_name == "hashing":
|
||||
stats.hashing_time += elapsed
|
||||
elif stage_name == "cache_lookup":
|
||||
stats.cache_lookup_time += elapsed
|
||||
elif stage_name == "prompt_update":
|
||||
stats.prompt_update_time += elapsed
|
||||
stats.preprocessor_total_time += elapsed
|
||||
return stats_dict
|
||||
|
||||
|
||||
_T = TypeVar("_T")
|
||||
@@ -151,21 +98,6 @@ class InputProcessingContext:
|
||||
tokenizer: TokenizerLike | None
|
||||
"""The tokenizer used to tokenize the inputs."""
|
||||
|
||||
observability_config: "ObservabilityConfig | None" = field(
|
||||
default=None, compare=False, repr=False
|
||||
)
|
||||
"""Configuration for observability features."""
|
||||
|
||||
timing_stats_registry: dict[str, MultiModalProcessorTimingStats] = field(
|
||||
default_factory=dict, compare=False, repr=False
|
||||
)
|
||||
"""Registry for storing timing stats keyed by request_id."""
|
||||
|
||||
_timing_stats_registry_lock: threading.Lock = field(
|
||||
default_factory=threading.Lock, compare=False, repr=False
|
||||
)
|
||||
"""Lock for thread-safe access to timing_stats_registry."""
|
||||
|
||||
def get_tokenizer(self) -> TokenizerLike:
|
||||
if self.tokenizer is None:
|
||||
raise ValueError(
|
||||
@@ -379,71 +311,6 @@ class InputProcessingContext:
|
||||
|
||||
return self._postprocess_output(output)
|
||||
|
||||
def get_timing_stats(
|
||||
self, request_id: str
|
||||
) -> MultiModalProcessorTimingStats | None:
|
||||
"""
|
||||
Get timing stats for a request.
|
||||
"""
|
||||
if (
|
||||
self.observability_config is None
|
||||
or not self.observability_config.enable_mm_processor_stats
|
||||
):
|
||||
return None
|
||||
with self._timing_stats_registry_lock:
|
||||
return self.timing_stats_registry.get(request_id)
|
||||
|
||||
def create_timing_stats(self, request_id: str) -> MultiModalProcessorTimingStats:
|
||||
"""
|
||||
Create and store timing stats in the registry for a request.
|
||||
|
||||
This should be called at the start of processing for a request.
|
||||
The stats object is created immediately and stored in the registry.
|
||||
"""
|
||||
if (
|
||||
self.observability_config is None
|
||||
or not self.observability_config.enable_mm_processor_stats
|
||||
):
|
||||
return MultiModalProcessorTimingStats()
|
||||
|
||||
with self._timing_stats_registry_lock:
|
||||
if request_id in self.timing_stats_registry:
|
||||
raise ValueError(
|
||||
f"Timing stats already exist for request_id: {request_id}"
|
||||
)
|
||||
stats = MultiModalProcessorTimingStats()
|
||||
self.timing_stats_registry[request_id] = stats
|
||||
return stats
|
||||
|
||||
def clear_timing_stats_registry(self) -> int:
|
||||
"""
|
||||
Clear all stats from the registry. Returns the number of stats cleared.
|
||||
"""
|
||||
if (
|
||||
self.observability_config is None
|
||||
or not self.observability_config.enable_mm_processor_stats
|
||||
):
|
||||
return 0
|
||||
with self._timing_stats_registry_lock:
|
||||
count = len(self.timing_stats_registry)
|
||||
self.timing_stats_registry.clear()
|
||||
return count
|
||||
|
||||
def get_all_timing_stats(self) -> dict[str, dict[str, float]]:
|
||||
"""
|
||||
Get all timing stats as a dictionary for API endpoints.
|
||||
"""
|
||||
if (
|
||||
self.observability_config is None
|
||||
or not self.observability_config.enable_mm_processor_stats
|
||||
):
|
||||
return {}
|
||||
with self._timing_stats_registry_lock:
|
||||
return {
|
||||
rid: stats.to_dict()
|
||||
for rid, stats in self.timing_stats_registry.items()
|
||||
}
|
||||
|
||||
|
||||
class BaseProcessingInfo:
|
||||
"""Base class to provide the information necessary for data processing."""
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Generic, TypeVar
|
||||
|
||||
import numpy as np
|
||||
@@ -18,27 +17,14 @@ from vllm.config.multimodal import (
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from ..inputs import MultiModalDataDict
|
||||
from ..parse import MultiModalDataItems
|
||||
from .context import BaseProcessingInfo
|
||||
from .inputs import ProcessorInputs
|
||||
|
||||
_I = TypeVar("_I", bound=BaseProcessingInfo)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProcessorInputs:
|
||||
"""
|
||||
Represents the keyword arguments to
|
||||
[`vllm.multimodal.processing.BaseMultiModalProcessor.apply`][].
|
||||
"""
|
||||
|
||||
prompt: str | list[int]
|
||||
mm_items: MultiModalDataItems
|
||||
hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)
|
||||
tokenization_kwargs: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
|
||||
class BaseDummyInputsBuilder(ABC, Generic[_I]):
|
||||
"""
|
||||
Abstract base class that constructs the dummy data to profile
|
||||
@@ -101,7 +87,7 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
|
||||
|
||||
return ProcessorInputs(
|
||||
prompt=dummy_text,
|
||||
mm_items=dummy_mm_items,
|
||||
mm_data_items=dummy_mm_items,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from ..hasher import MultiModalHasher
|
||||
from ..inputs import MultiModalHashes
|
||||
from ..parse import MultiModalDataItems, MultiModalUUIDItems
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProcessorInputs:
|
||||
"""
|
||||
Represents the keyword arguments to
|
||||
[`vllm.multimodal.processing.BaseMultiModalProcessor.apply`][].
|
||||
"""
|
||||
|
||||
prompt: str | list[int]
|
||||
mm_data_items: MultiModalDataItems
|
||||
mm_uuid_items: MultiModalUUIDItems | None = None
|
||||
hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)
|
||||
tokenization_kwargs: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
def get_mm_hashes(self, model_id: str) -> MultiModalHashes:
|
||||
mm_data_items = self.mm_data_items
|
||||
mm_uuid_items = self.mm_uuid_items or {}
|
||||
hf_processor_mm_kwargs = self.hf_processor_mm_kwargs
|
||||
|
||||
mm_hashes: MultiModalHashes = {}
|
||||
hasher = MultiModalHasher
|
||||
|
||||
for modality, data_items in mm_data_items.items():
|
||||
if modality in mm_uuid_items:
|
||||
uuid_items = mm_uuid_items[modality]
|
||||
|
||||
# For None entries, compute a hash; otherwise, use provided ID.
|
||||
hashes: list[str] = []
|
||||
for i, item in enumerate(data_items.get_all_items_for_hash()):
|
||||
uuid_item = uuid_items[i]
|
||||
|
||||
# NOTE: Even if a uuid_item is provided, we still compute a hash
|
||||
# if `hf_processor_mm_kwargs` is provided.
|
||||
# This is because the processed multimodal inputs can be different
|
||||
# depending on the processor kwargs.
|
||||
if uuid_item is None or hf_processor_mm_kwargs:
|
||||
# NOTE: use provided hash string to hash with kwargs
|
||||
# if available for better performance.
|
||||
item = uuid_item if uuid_item is not None else item
|
||||
hashes.append(
|
||||
hasher.hash_kwargs(
|
||||
model_id=model_id,
|
||||
**{modality: item},
|
||||
**hf_processor_mm_kwargs,
|
||||
)
|
||||
)
|
||||
else:
|
||||
hashes.append(uuid_item)
|
||||
|
||||
mm_hashes[modality] = hashes
|
||||
else:
|
||||
mm_hashes[modality] = [
|
||||
hasher.hash_kwargs(
|
||||
model_id=model_id,
|
||||
**{modality: item},
|
||||
**hf_processor_mm_kwargs,
|
||||
)
|
||||
for item in data_items
|
||||
]
|
||||
|
||||
return mm_hashes
|
||||
@@ -23,7 +23,6 @@ from vllm.logger import init_logger
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.utils.collection_utils import flatten_2d_lists, full_groupby
|
||||
|
||||
from ..hasher import MultiModalHasher
|
||||
from ..inputs import (
|
||||
MultiModalEncDecInputs,
|
||||
MultiModalFieldConfig,
|
||||
@@ -42,12 +41,9 @@ from ..parse import (
|
||||
MultiModalDataItems,
|
||||
MultiModalUUIDItems,
|
||||
)
|
||||
from .context import (
|
||||
BaseProcessingInfo,
|
||||
get_current_request_id,
|
||||
timed_preprocessor_operation,
|
||||
)
|
||||
from .context import BaseProcessingInfo, TimingContext
|
||||
from .dummy_inputs import BaseDummyInputsBuilder
|
||||
from .inputs import ProcessorInputs
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
@@ -1017,13 +1013,15 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
mm_uuid_items: MultiModalUUIDItems | None = None,
|
||||
hf_processor_mm_kwargs: Mapping[str, object] | None = None,
|
||||
) -> MultiModalInputs:
|
||||
return self.apply(
|
||||
processor_inputs = ProcessorInputs(
|
||||
prompt,
|
||||
mm_items,
|
||||
mm_uuid_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs or {},
|
||||
)
|
||||
|
||||
return self.apply(processor_inputs, TimingContext(enabled=False))
|
||||
|
||||
@abstractmethod
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
@@ -1139,12 +1137,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
Call the HF processor on the prompt text and
|
||||
associated multi-modal data.
|
||||
"""
|
||||
with timed_preprocessor_operation(self.info.ctx, "hf_processor"):
|
||||
return self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**mm_kwargs),
|
||||
dict(text=prompt, **mm_data),
|
||||
dict(**mm_kwargs, **tok_kwargs),
|
||||
)
|
||||
return self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**mm_kwargs),
|
||||
dict(text=prompt, **mm_data),
|
||||
dict(**mm_kwargs, **tok_kwargs),
|
||||
)
|
||||
|
||||
def _hf_processor_applies_updates(
|
||||
self,
|
||||
@@ -1306,60 +1303,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
|
||||
return prompt_ids, mm_processed_data, False
|
||||
|
||||
def _hash_mm_items(
|
||||
self,
|
||||
mm_data_items: MultiModalDataItems,
|
||||
mm_uuid_items: MultiModalUUIDItems | None,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> MultiModalHashes:
|
||||
model_id = self.info.model_id
|
||||
|
||||
if mm_uuid_items is None:
|
||||
mm_uuid_items = {}
|
||||
|
||||
mm_hashes: MultiModalHashes = {}
|
||||
hasher = MultiModalHasher
|
||||
|
||||
for modality, data_items in mm_data_items.items():
|
||||
if modality in mm_uuid_items:
|
||||
uuid_items = mm_uuid_items[modality]
|
||||
|
||||
# For None entries, compute a hash; otherwise, use provided ID.
|
||||
hashes: list[str] = []
|
||||
for i, item in enumerate(data_items.get_all_items_for_hash()):
|
||||
uuid_item = uuid_items[i]
|
||||
|
||||
# NOTE: Even if a uuid_item is provided, we still compute a hash
|
||||
# if `hf_processor_mm_kwargs` is provided.
|
||||
# This is because the processed multimodal inputs can be different
|
||||
# depending on the processor kwargs.
|
||||
if uuid_item is None or hf_processor_mm_kwargs:
|
||||
# NOTE: use provided hash string to hash with kwargs
|
||||
# if available for better performance.
|
||||
item = uuid_item if uuid_item is not None else item
|
||||
hashes.append(
|
||||
hasher.hash_kwargs(
|
||||
model_id=model_id,
|
||||
**{modality: item},
|
||||
**hf_processor_mm_kwargs,
|
||||
)
|
||||
)
|
||||
else:
|
||||
hashes.append(uuid_item)
|
||||
|
||||
mm_hashes[modality] = hashes
|
||||
else:
|
||||
mm_hashes[modality] = [
|
||||
hasher.hash_kwargs(
|
||||
model_id=model_id,
|
||||
**{modality: item},
|
||||
**hf_processor_mm_kwargs,
|
||||
)
|
||||
for item in data_items
|
||||
]
|
||||
|
||||
return mm_hashes
|
||||
|
||||
def _get_cache_missing_items(
|
||||
self,
|
||||
cache: BaseMultiModalProcessorCache,
|
||||
@@ -1461,40 +1404,36 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
|
||||
def _apply_hf_processor(
|
||||
self,
|
||||
prompt: str | list[int],
|
||||
mm_data_items: MultiModalDataItems,
|
||||
mm_uuid_items: MultiModalUUIDItems | None,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
|
||||
(
|
||||
prompt_ids,
|
||||
mm_processed_data,
|
||||
is_update_applied,
|
||||
) = self._apply_hf_processor_main(
|
||||
prompt=prompt,
|
||||
mm_items=mm_data_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
enable_hf_prompt_update=True,
|
||||
)
|
||||
with timing_ctx.record("apply_hf_processor"):
|
||||
(
|
||||
prompt_ids,
|
||||
mm_processed_data,
|
||||
is_update_applied,
|
||||
) = self._apply_hf_processor_main(
|
||||
prompt=inputs.prompt,
|
||||
mm_items=inputs.mm_data_items,
|
||||
hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
|
||||
tokenization_kwargs=inputs.tokenization_kwargs,
|
||||
enable_hf_prompt_update=True,
|
||||
)
|
||||
|
||||
mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
|
||||
mm_processed_data,
|
||||
self._get_mm_fields_config(mm_processed_data, hf_processor_mm_kwargs),
|
||||
self._get_mm_fields_config(
|
||||
mm_processed_data, inputs.hf_processor_mm_kwargs
|
||||
),
|
||||
)
|
||||
|
||||
# Use overrides if provided; fallback to data-dependent hashing.
|
||||
with timed_preprocessor_operation(self.info.ctx, "hashing"):
|
||||
mm_hashes = self._hash_mm_items(
|
||||
mm_data_items,
|
||||
mm_uuid_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
with timing_ctx.record("get_mm_hashes"):
|
||||
mm_hashes = inputs.get_mm_hashes(self.info.model_id)
|
||||
|
||||
mm_prompt_updates = self._get_mm_prompt_updates(
|
||||
mm_data_items,
|
||||
hf_processor_mm_kwargs,
|
||||
inputs.mm_data_items,
|
||||
inputs.hf_processor_mm_kwargs,
|
||||
mm_kwargs,
|
||||
)
|
||||
|
||||
@@ -1508,11 +1447,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
|
||||
def _cached_apply_hf_processor(
|
||||
self,
|
||||
prompt: str | list[int],
|
||||
mm_data_items: MultiModalDataItems,
|
||||
mm_uuid_items: MultiModalUUIDItems | None,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
|
||||
"""
|
||||
Apply the HF processor on the full prompt text,
|
||||
@@ -1520,59 +1456,50 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
"""
|
||||
cache = self.cache
|
||||
|
||||
_, passthrough_data = self._get_hf_mm_data(mm_data_items)
|
||||
_, passthrough_data = self._get_hf_mm_data(inputs.mm_data_items)
|
||||
if cache is None or passthrough_data:
|
||||
return self._apply_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data_items=mm_data_items,
|
||||
mm_uuid_items=mm_uuid_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
return self._apply_hf_processor(inputs, timing_ctx)
|
||||
|
||||
with timed_preprocessor_operation(self.info.ctx, "hashing"):
|
||||
mm_hashes = self._hash_mm_items(
|
||||
mm_data_items,
|
||||
mm_uuid_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
with timing_ctx.record("get_mm_hashes"):
|
||||
mm_hashes = inputs.get_mm_hashes(self.info.model_id)
|
||||
|
||||
with timed_preprocessor_operation(self.info.ctx, "cache_lookup"):
|
||||
with timing_ctx.record("get_cache_missing_items"):
|
||||
mm_is_cached, mm_missing_data_items = self._get_cache_missing_items(
|
||||
cache=cache,
|
||||
mm_data_items=mm_data_items,
|
||||
mm_data_items=inputs.mm_data_items,
|
||||
mm_hashes=mm_hashes,
|
||||
)
|
||||
|
||||
# NOTE: `prompt` does not correspond to `mm_missing_data_items`,
|
||||
# so we can't apply prompt updates until the new multimodal
|
||||
# items are combined with the cached multimodal items
|
||||
(
|
||||
prompt_ids,
|
||||
mm_missing_processed_data,
|
||||
is_update_applied,
|
||||
) = self._apply_hf_processor_main(
|
||||
prompt=prompt,
|
||||
mm_items=mm_missing_data_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
enable_hf_prompt_update=False,
|
||||
)
|
||||
with timing_ctx.record("apply_hf_processor"):
|
||||
(
|
||||
prompt_ids,
|
||||
mm_missing_processed_data,
|
||||
is_update_applied,
|
||||
) = self._apply_hf_processor_main(
|
||||
prompt=inputs.prompt,
|
||||
mm_items=mm_missing_data_items,
|
||||
hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
|
||||
tokenization_kwargs=inputs.tokenization_kwargs,
|
||||
enable_hf_prompt_update=False,
|
||||
)
|
||||
|
||||
mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs(
|
||||
mm_missing_processed_data,
|
||||
self._get_mm_fields_config(
|
||||
mm_missing_processed_data, hf_processor_mm_kwargs
|
||||
mm_missing_processed_data, inputs.hf_processor_mm_kwargs
|
||||
),
|
||||
)
|
||||
|
||||
mm_missing_prompt_updates = self._get_mm_prompt_updates(
|
||||
mm_missing_data_items,
|
||||
hf_processor_mm_kwargs,
|
||||
inputs.hf_processor_mm_kwargs,
|
||||
mm_missing_kwargs,
|
||||
)
|
||||
|
||||
with timed_preprocessor_operation(self.info.ctx, "cache_lookup"):
|
||||
with timing_ctx.record("merge_mm_kwargs"):
|
||||
mm_kwargs, mm_prompt_updates = self._merge_mm_kwargs(
|
||||
cache,
|
||||
mm_hashes=mm_hashes,
|
||||
@@ -1742,11 +1669,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
|
||||
def apply(
|
||||
self,
|
||||
prompt: str | list[int],
|
||||
mm_items: MultiModalDataItems,
|
||||
mm_uuid_items: MultiModalUUIDItems | None = None,
|
||||
hf_processor_mm_kwargs: Mapping[str, object] | None = None,
|
||||
tokenization_kwargs: Mapping[str, object] | None = None,
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> MultiModalInputs:
|
||||
"""
|
||||
Process multi-modal inputs to be used in vLLM.
|
||||
@@ -1761,31 +1685,16 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
3. Extract information about the placeholder tokens from the
|
||||
processed token IDs.
|
||||
"""
|
||||
request_id = get_current_request_id()
|
||||
if request_id is not None:
|
||||
self.info.ctx.create_timing_stats(request_id)
|
||||
|
||||
if hf_processor_mm_kwargs is None:
|
||||
hf_processor_mm_kwargs = {}
|
||||
if tokenization_kwargs is None:
|
||||
tokenization_kwargs = {}
|
||||
|
||||
(
|
||||
prompt_ids,
|
||||
mm_info,
|
||||
is_update_applied,
|
||||
) = self._cached_apply_hf_processor(
|
||||
prompt,
|
||||
mm_items,
|
||||
mm_uuid_items,
|
||||
hf_processor_mm_kwargs,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
) = self._cached_apply_hf_processor(inputs, timing_ctx)
|
||||
|
||||
# NOTE: tokenization_kwargs are not required to init processor
|
||||
with timed_preprocessor_operation(self.info.ctx, "prompt_update"):
|
||||
with timing_ctx.record("apply_prompt_updates"):
|
||||
prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
|
||||
mm_items=mm_items,
|
||||
mm_items=inputs.mm_data_items,
|
||||
prompt_ids=prompt_ids,
|
||||
mm_kwargs=mm_info.kwargs,
|
||||
mm_prompt_updates=mm_info.prompt_updates,
|
||||
@@ -1851,11 +1760,8 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
|
||||
def apply(
|
||||
self,
|
||||
prompt: str | list[int],
|
||||
mm_items: MultiModalDataItems,
|
||||
mm_uuid_items: MultiModalUUIDItems | None = None,
|
||||
hf_processor_mm_kwargs: Mapping[str, object] | None = None,
|
||||
tokenization_kwargs: Mapping[str, object] | None = None,
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> MultiModalEncDecInputs:
|
||||
"""
|
||||
Process multi-modal inputs to be used in vLLM.
|
||||
@@ -1864,17 +1770,22 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
2. Apply the HF processor on encoder prompt.
|
||||
3. Copy the input prompt text as decoder prompt inputs.
|
||||
"""
|
||||
encoder_prompt = self.create_encoder_prompt(prompt, mm_items)
|
||||
encoder_inputs = super().apply(
|
||||
encoder_prompt = self.create_encoder_prompt(
|
||||
inputs.prompt,
|
||||
inputs.mm_data_items,
|
||||
)
|
||||
encoder_processor_inputs = ProcessorInputs(
|
||||
encoder_prompt,
|
||||
mm_items,
|
||||
mm_uuid_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
inputs.mm_data_items,
|
||||
inputs.mm_uuid_items,
|
||||
hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
|
||||
tokenization_kwargs=inputs.tokenization_kwargs,
|
||||
)
|
||||
|
||||
encoder_inputs = super().apply(encoder_processor_inputs, timing_ctx)
|
||||
|
||||
return self._get_enc_dec_inputs(
|
||||
prompt=prompt,
|
||||
mm_items=mm_items,
|
||||
prompt=inputs.prompt,
|
||||
mm_items=inputs.mm_data_items,
|
||||
encoder_inputs=encoder_inputs,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user