[Refactor] Move profiling methods to MM budget (#33559)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -0,0 +1,154 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Mapping
|
||||
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor
|
||||
from vllm.multimodal.registry import MultiModalRegistry
|
||||
from vllm.utils.torch_utils import set_default_torch_num_threads
|
||||
from vllm.v1.core.encoder_cache_manager import compute_mm_encoder_budget
|
||||
|
||||
|
||||
def get_mm_max_toks_per_item(
|
||||
model_config: ModelConfig,
|
||||
mm_registry: MultiModalRegistry,
|
||||
processor: BaseMultiModalProcessor,
|
||||
mm_counts: Mapping[str, int],
|
||||
) -> Mapping[str, int]:
|
||||
"""
|
||||
Get the maximum number of tokens per data item from each modality based
|
||||
on underlying model configuration.
|
||||
"""
|
||||
max_tokens_per_item = processor.info.get_mm_max_tokens_per_item(
|
||||
seq_len=model_config.max_model_len,
|
||||
mm_counts=mm_counts,
|
||||
)
|
||||
if max_tokens_per_item is not None:
|
||||
return max_tokens_per_item
|
||||
|
||||
mm_inputs = mm_registry.get_dummy_mm_inputs(
|
||||
model_config,
|
||||
mm_counts=mm_counts,
|
||||
processor=processor,
|
||||
)
|
||||
|
||||
return {
|
||||
modality: sum(item.get_num_embeds for item in placeholders)
|
||||
for modality, placeholders in mm_inputs["mm_placeholders"].items()
|
||||
}
|
||||
|
||||
|
||||
class MultiModalBudget:
|
||||
"""Helper class to calculate budget information for multi-modal models."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
mm_registry: MultiModalRegistry,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.model_config = model_config = vllm_config.model_config
|
||||
self.scheduler_config = scheduler_config = vllm_config.scheduler_config
|
||||
|
||||
self.max_model_len = model_config.max_model_len
|
||||
self.max_num_reqs = scheduler_config.max_num_seqs
|
||||
|
||||
cache = mm_registry.processor_only_cache_from_config(vllm_config)
|
||||
processor = mm_registry.create_processor(model_config, cache=cache)
|
||||
|
||||
self.cache = cache
|
||||
self.mm_limits = mm_limits = processor.info.allowed_mm_limits
|
||||
|
||||
active_modalities = {
|
||||
modality for modality, limit in mm_limits.items() if limit > 0
|
||||
}
|
||||
|
||||
with set_default_torch_num_threads(): # Avoid hang during startup
|
||||
all_mm_max_toks_per_item = get_mm_max_toks_per_item(
|
||||
model_config,
|
||||
mm_registry,
|
||||
processor,
|
||||
mm_counts=dict.fromkeys(active_modalities, 1),
|
||||
)
|
||||
|
||||
mm_max_toks_per_item = {
|
||||
modality: all_mm_max_toks_per_item[modality]
|
||||
for modality in active_modalities
|
||||
}
|
||||
|
||||
encoder_compute_budget, encoder_cache_size = compute_mm_encoder_budget(
|
||||
scheduler_config,
|
||||
mm_max_toks_per_item,
|
||||
)
|
||||
|
||||
self.encoder_compute_budget = encoder_compute_budget
|
||||
self.encoder_cache_size = encoder_cache_size
|
||||
|
||||
mm_max_items_per_prompt = dict[str, int]()
|
||||
mm_max_items_per_batch = dict[str, int]()
|
||||
|
||||
for modality, max_toks_per_item in mm_max_toks_per_item.items():
|
||||
(
|
||||
mm_max_items_per_prompt[modality],
|
||||
mm_max_items_per_batch[modality],
|
||||
) = self._get_max_items(modality, max_toks_per_item)
|
||||
|
||||
self.mm_max_toks_per_item = mm_max_toks_per_item
|
||||
self.mm_max_items_per_prompt: Mapping[str, int] = mm_max_items_per_prompt
|
||||
self.mm_max_items_per_batch: Mapping[str, int] = mm_max_items_per_batch
|
||||
|
||||
def _get_max_items(
|
||||
self,
|
||||
modality: str,
|
||||
max_tokens_per_item: int,
|
||||
) -> tuple[int, int]:
|
||||
if max_tokens_per_item == 0:
|
||||
return 0, 0
|
||||
|
||||
# Check how many items of this modality can be supported by
|
||||
# the encoder budget.
|
||||
if (encoder_budget := self.get_encoder_budget()) == 0:
|
||||
return 0, 0
|
||||
|
||||
max_encoder_items_per_batch = encoder_budget // max_tokens_per_item
|
||||
|
||||
# Check how many items of this modality can be supported by
|
||||
# the decoder budget.
|
||||
mm_limit = self.mm_limits[modality]
|
||||
|
||||
max_items_per_prompt = max(
|
||||
1,
|
||||
min(mm_limit, self.max_model_len // max_tokens_per_item),
|
||||
)
|
||||
|
||||
scheduler_config = self.scheduler_config
|
||||
max_num_reqs = self.max_num_reqs
|
||||
|
||||
if not scheduler_config.enable_chunked_prefill:
|
||||
max_num_reqs = min(
|
||||
max_num_reqs,
|
||||
scheduler_config.max_num_batched_tokens // max_tokens_per_item,
|
||||
)
|
||||
|
||||
max_decoder_items_per_batch = max_num_reqs * max_items_per_prompt
|
||||
|
||||
max_items_per_batch = max(
|
||||
1,
|
||||
min(max_encoder_items_per_batch, max_decoder_items_per_batch),
|
||||
)
|
||||
|
||||
return max_items_per_prompt, max_items_per_batch
|
||||
|
||||
def get_modality_with_max_tokens(self) -> str:
|
||||
mm_max_toks_per_item = self.mm_max_toks_per_item
|
||||
modality, _ = max(mm_max_toks_per_item.items(), key=lambda x: x[1])
|
||||
|
||||
return modality
|
||||
|
||||
def get_encoder_budget(self) -> int:
|
||||
return min(self.encoder_compute_budget, self.encoder_cache_size)
|
||||
|
||||
def reset_cache(self) -> None:
|
||||
if self.cache is not None:
|
||||
self.cache.clear_cache()
|
||||
@@ -130,15 +130,13 @@ class MultiModalRegistry:
|
||||
if not model_config.is_multimodal_model:
|
||||
return False
|
||||
|
||||
info = self._create_processing_info(model_config, tokenizer=None)
|
||||
supported_modalities = info.get_supported_mm_limits()
|
||||
|
||||
mm_config = model_config.get_multimodal_config()
|
||||
info = self._create_processing_info(model_config, tokenizer=None)
|
||||
|
||||
# Check if all supported modalities have limit == 0
|
||||
if all(
|
||||
mm_config.get_limit_per_prompt(modality) == 0
|
||||
for modality in supported_modalities
|
||||
for modality in info.supported_mm_limits
|
||||
):
|
||||
logger.info_once(
|
||||
"All limits of multimodal modalities supported by the model "
|
||||
@@ -148,70 +146,6 @@ class MultiModalRegistry:
|
||||
|
||||
return True
|
||||
|
||||
def get_max_tokens_per_item_by_modality(
|
||||
self,
|
||||
model_config: "ModelConfig",
|
||||
*,
|
||||
cache: BaseMultiModalProcessorCache | None = None,
|
||||
profiler_limits: Mapping[str, int] | None = None,
|
||||
observability_config: ObservabilityConfig | None = None,
|
||||
) -> Mapping[str, int]:
|
||||
"""
|
||||
Get the maximum number of tokens per data item from each modality based
|
||||
on underlying model configuration.
|
||||
"""
|
||||
if not model_config.is_multimodal_model:
|
||||
return {}
|
||||
|
||||
processor = self.create_processor(
|
||||
model_config, observability_config, cache=cache
|
||||
)
|
||||
|
||||
if profiler_limits is None:
|
||||
profiler_limits = processor.info.allowed_mm_limits
|
||||
|
||||
mm_counts = {
|
||||
modality: 1 for modality, limit in profiler_limits.items() if limit > 0
|
||||
}
|
||||
|
||||
max_tokens_per_item = processor.info.get_mm_max_tokens_per_item(
|
||||
seq_len=model_config.max_model_len,
|
||||
mm_counts=mm_counts,
|
||||
)
|
||||
if max_tokens_per_item is not None:
|
||||
return {
|
||||
modality: max_tokens
|
||||
for modality, max_tokens in max_tokens_per_item.items()
|
||||
if mm_counts.get(modality, 0) > 0
|
||||
}
|
||||
|
||||
mm_inputs = self.get_dummy_mm_inputs(
|
||||
model_config,
|
||||
mm_counts=mm_counts,
|
||||
processor=processor,
|
||||
)
|
||||
|
||||
return {
|
||||
modality: sum(item.get_num_embeds for item in placeholders)
|
||||
for modality, placeholders in mm_inputs["mm_placeholders"].items()
|
||||
}
|
||||
|
||||
def get_mm_limits_per_prompt(
|
||||
self,
|
||||
model_config: "ModelConfig",
|
||||
*,
|
||||
observability_config: ObservabilityConfig | None = None,
|
||||
) -> Mapping[str, int]:
|
||||
"""
|
||||
Get the maximum number of multi-modal input instances for each modality
|
||||
that are allowed per prompt for a model class.
|
||||
"""
|
||||
if not model_config.is_multimodal_model:
|
||||
return {}
|
||||
|
||||
info = self._create_processing_info(model_config, observability_config)
|
||||
return info.allowed_mm_limits
|
||||
|
||||
def register_processor(
|
||||
self,
|
||||
processor: MultiModalProcessorFactory[_I],
|
||||
@@ -303,10 +237,9 @@ class MultiModalRegistry:
|
||||
def get_dummy_mm_inputs(
|
||||
self,
|
||||
model_config: "ModelConfig",
|
||||
mm_counts: Mapping[str, int] | None = None,
|
||||
mm_counts: Mapping[str, int],
|
||||
*,
|
||||
cache: BaseMultiModalProcessorCache | None = None,
|
||||
observability_config: ObservabilityConfig | None = None,
|
||||
processor: BaseMultiModalProcessor | None = None,
|
||||
) -> MultiModalInputs:
|
||||
"""
|
||||
@@ -317,11 +250,7 @@ class MultiModalRegistry:
|
||||
seq_len = model_config.max_model_len
|
||||
|
||||
if processor is None:
|
||||
processor = self.create_processor(
|
||||
model_config, observability_config, cache=cache
|
||||
)
|
||||
if mm_counts is None:
|
||||
mm_counts = processor.info.allowed_mm_limits
|
||||
processor = self.create_processor(model_config, cache=cache)
|
||||
|
||||
processor_inputs = processor.dummy_inputs.get_dummy_processor_inputs(
|
||||
seq_len=seq_len,
|
||||
@@ -342,26 +271,6 @@ class MultiModalRegistry:
|
||||
|
||||
return mm_inputs
|
||||
|
||||
def get_encdec_max_encoder_len(self, model_config: "ModelConfig") -> int:
|
||||
"""
|
||||
Get the maximum length of the encoder input for encoder-decoder models.
|
||||
"""
|
||||
if not model_config.is_encoder_decoder:
|
||||
return 0
|
||||
max_tokens = self.get_max_tokens_per_item_by_modality(model_config)
|
||||
if not max_tokens:
|
||||
# TODO - this function assumes encoder-decoder models are
|
||||
# multimodal. This will need to change when adding support for more
|
||||
# than whisper.
|
||||
return 0
|
||||
assert len(max_tokens) == 1, (
|
||||
"Encoder-decoder models are expected "
|
||||
"to implement the multimodal interface with at most one modality."
|
||||
)
|
||||
|
||||
first_modality = next(iter(max_tokens))
|
||||
return max_tokens[first_modality]
|
||||
|
||||
def _get_cache_type(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
|
||||
Reference in New Issue
Block a user