This commit is contained in:
@@ -24,7 +24,6 @@ from vllm.config.multimodal import MultiModalConfig
|
||||
from vllm.config.observability import ObservabilityConfig
|
||||
from vllm.config.parallel import EPLBConfig, ParallelConfig
|
||||
from vllm.config.pooler import PoolerConfig
|
||||
from vllm.config.renderer import RendererConfig
|
||||
from vllm.config.scheduler import SchedulerConfig
|
||||
from vllm.config.speculative import SpeculativeConfig
|
||||
from vllm.config.speech_to_text import SpeechToTextConfig
|
||||
@@ -82,8 +81,6 @@ __all__ = [
|
||||
"ParallelConfig",
|
||||
# From vllm.config.pooler
|
||||
"PoolerConfig",
|
||||
# From vllm.config.renderer
|
||||
"RendererConfig",
|
||||
# From vllm.config.scheduler
|
||||
"SchedulerConfig",
|
||||
# From vllm.config.speculative
|
||||
|
||||
+110
-31
@@ -36,6 +36,7 @@ from vllm.transformers_utils.config import (
|
||||
uses_xdrope_dim,
|
||||
)
|
||||
from vllm.transformers_utils.gguf_utils import (
|
||||
is_gguf,
|
||||
is_remote_gguf,
|
||||
maybe_patch_hf_config_from_gguf,
|
||||
split_remote_gguf,
|
||||
@@ -82,6 +83,7 @@ TaskOption = Literal[
|
||||
"transcription",
|
||||
"draft",
|
||||
]
|
||||
TokenizerMode = Literal["auto", "hf", "slow", "mistral", "deepseek_v32"]
|
||||
ModelDType = Literal["auto", "half", "float16", "bfloat16", "float", "float32"]
|
||||
LogprobsMode = Literal[
|
||||
"raw_logits", "raw_logprobs", "processed_logits", "processed_logprobs"
|
||||
@@ -129,6 +131,18 @@ class ModelConfig:
|
||||
|
||||
Note that the model may support other tasks using the same model runner.
|
||||
"""
|
||||
tokenizer: SkipValidation[str] = None # type: ignore
|
||||
"""Name or path of the Hugging Face tokenizer to use. If unspecified, model
|
||||
name or path will be used."""
|
||||
tokenizer_mode: TokenizerMode | str = "auto"
|
||||
"""Tokenizer mode:\n
|
||||
- "auto" will use the tokenizer from `mistral_common` for Mistral models
|
||||
if available, otherwise it will use the "hf" tokenizer.\n
|
||||
- "hf" will use the fast tokenizer if available.\n
|
||||
- "slow" will always use the slow tokenizer.\n
|
||||
- "mistral" will always use the tokenizer from `mistral_common`.\n
|
||||
- "deepseek_v32" will always use the tokenizer from `deepseek_v32`.\n
|
||||
- Other custom values can be supported via plugins."""
|
||||
trust_remote_code: bool = False
|
||||
"""Trust remote code (e.g., from HuggingFace) when downloading the model
|
||||
and tokenizer."""
|
||||
@@ -154,6 +168,13 @@ class ModelConfig:
|
||||
hf_config_path: str | None = None
|
||||
"""Name or path of the Hugging Face config to use. If unspecified, model
|
||||
name or path will be used."""
|
||||
allowed_local_media_path: str = ""
|
||||
"""Allowing API requests to read local images or videos from directories
|
||||
specified by the server file system. This is a security risk. Should only
|
||||
be enabled in trusted environments."""
|
||||
allowed_media_domains: list[str] | None = None
|
||||
"""If set, only media URLs that belong to this domain can be used for
|
||||
multi-modal inputs. """
|
||||
revision: str | None = None
|
||||
"""The specific model version to use. It can be a branch name, a tag name,
|
||||
or a commit id. If unspecified, will use the default version."""
|
||||
@@ -161,6 +182,10 @@ class ModelConfig:
|
||||
"""The specific revision to use for the model code on the Hugging Face Hub.
|
||||
It can be a branch name, a tag name, or a commit id. If unspecified, will
|
||||
use the default version."""
|
||||
tokenizer_revision: str | None = None
|
||||
"""The specific revision to use for the tokenizer on the Hugging Face Hub.
|
||||
It can be a branch name, a tag name, or a commit id. If unspecified, will
|
||||
use the default version."""
|
||||
max_model_len: SkipValidation[int] = None # type: ignore
|
||||
"""Model context length (prompt and output). If unspecified, will be
|
||||
automatically derived from the model config.
|
||||
@@ -205,6 +230,10 @@ class ModelConfig:
|
||||
preventing potential numerical issues. Note that even if this is set to
|
||||
False, cascade attention will be only used when the heuristic tells that
|
||||
it's beneficial."""
|
||||
skip_tokenizer_init: bool = False
|
||||
"""Skip initialization of tokenizer and detokenizer. Expects valid
|
||||
`prompt_token_ids` and `None` for prompt from the input. The generated
|
||||
output will contain token ids."""
|
||||
enable_prompt_embeds: bool = False
|
||||
"""If `True`, enables passing text embeddings as inputs via the
|
||||
`prompt_embeds` key.
|
||||
@@ -265,6 +294,8 @@ class ModelConfig:
|
||||
logits_processors: list[str | type[LogitsProcessor]] | None = None
|
||||
"""One or more logits processors' fully-qualified class names or class
|
||||
definitions"""
|
||||
io_processor_plugin: str | None = None
|
||||
"""IOProcessor plugin name to load at model startup"""
|
||||
|
||||
# Pooler config
|
||||
pooler_config: PoolerConfig | None = None
|
||||
@@ -277,6 +308,7 @@ class ModelConfig:
|
||||
from the architecture of `self.model`."""
|
||||
limit_mm_per_prompt: InitVar[dict[str, int | dict[str, int]] | None] = None
|
||||
enable_mm_embeds: InitVar[bool | None] = None
|
||||
media_io_kwargs: InitVar[dict[str, dict[str, Any]] | None] = None
|
||||
mm_processor_kwargs: InitVar[dict[str, Any] | None] = None
|
||||
mm_processor_cache_gb: InitVar[float | None] = None
|
||||
mm_processor_cache_type: InitVar[MMCacheType | None] = None
|
||||
@@ -303,12 +335,18 @@ class ModelConfig:
|
||||
"runner",
|
||||
"convert",
|
||||
"task",
|
||||
"tokenizer",
|
||||
"tokenizer_mode",
|
||||
"seed",
|
||||
"hf_config_path",
|
||||
"allowed_local_media_path",
|
||||
"allowed_media_domains",
|
||||
"tokenizer_revision",
|
||||
"spec_target_max_model_len",
|
||||
"enforce_eager",
|
||||
"logprobs_mode",
|
||||
"disable_cascade_attn",
|
||||
"skip_tokenizer_init",
|
||||
"served_model_name",
|
||||
"config_format",
|
||||
"hf_token",
|
||||
@@ -316,9 +354,11 @@ class ModelConfig:
|
||||
"logits_processor_pattern",
|
||||
"override_attention_dtype",
|
||||
"logits_processors",
|
||||
"io_processor_plugin",
|
||||
"pooler_config",
|
||||
"multimodal_config",
|
||||
"limit_mm_per_prompt",
|
||||
"media_io_kwargs",
|
||||
"mm_processor_kwargs",
|
||||
"mm_processor_cache_gb",
|
||||
"mm_processor_cache_type",
|
||||
@@ -383,6 +423,7 @@ class ModelConfig:
|
||||
# Multimodal config init vars
|
||||
limit_mm_per_prompt: dict[str, int | dict[str, int]] | None,
|
||||
enable_mm_embeds: bool | None,
|
||||
media_io_kwargs: dict[str, dict[str, Any]] | None,
|
||||
mm_processor_kwargs: dict[str, Any] | None,
|
||||
mm_processor_cache_gb: float | None,
|
||||
mm_processor_cache_type: MMCacheType | None,
|
||||
@@ -397,8 +438,13 @@ class ModelConfig:
|
||||
self.served_model_name = get_served_model_name(
|
||||
self.model, self.served_model_name
|
||||
)
|
||||
self.original_model = self.model
|
||||
self.model = maybe_model_redirect(self.original_model)
|
||||
self.model = maybe_model_redirect(self.model)
|
||||
# The tokenizer is consistent with the model by default.
|
||||
if self.tokenizer is None:
|
||||
self.tokenizer = self.model
|
||||
if self.tokenizer_revision is None:
|
||||
self.tokenizer_revision = self.revision
|
||||
self.tokenizer = maybe_model_redirect(self.tokenizer)
|
||||
|
||||
if isinstance(self.hf_config_path, str):
|
||||
self.hf_config_path = maybe_model_redirect(self.hf_config_path)
|
||||
@@ -419,7 +465,7 @@ class ModelConfig:
|
||||
hf_overrides_kw[key] = value
|
||||
hf_overrides_fn = None
|
||||
|
||||
self.maybe_pull_model_for_runai(self.model)
|
||||
self.maybe_pull_model_tokenizer_for_runai(self.model, self.tokenizer)
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -602,8 +648,7 @@ class ModelConfig:
|
||||
)
|
||||
|
||||
self.original_max_model_len = self.max_model_len
|
||||
self.recalculate_max_model_len(self.original_max_model_len)
|
||||
|
||||
self.max_model_len = self.get_and_verify_max_len(self.max_model_len)
|
||||
# Init multimodal config if needed
|
||||
if self._model_info.supports_multimodal:
|
||||
if (
|
||||
@@ -619,6 +664,7 @@ class ModelConfig:
|
||||
mm_config_kwargs = dict(
|
||||
limit_per_prompt=limit_mm_per_prompt,
|
||||
enable_mm_embeds=enable_mm_embeds,
|
||||
media_io_kwargs=media_io_kwargs,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
mm_processor_cache_gb=mm_processor_cache_gb,
|
||||
mm_processor_cache_type=mm_processor_cache_type,
|
||||
@@ -636,8 +682,16 @@ class ModelConfig:
|
||||
|
||||
self.multimodal_config = MultiModalConfig(**mm_config_kwargs)
|
||||
|
||||
# Multimodal GGUF models must use original repo for mm processing
|
||||
if is_gguf(self.tokenizer) and self.is_multimodal_model:
|
||||
raise ValueError(
|
||||
"Loading a multimodal GGUF model needs to use original "
|
||||
"tokenizer. Please specify the unquantized hf model's "
|
||||
"repo name or path using the --tokenizer argument."
|
||||
)
|
||||
|
||||
if self.disable_sliding_window:
|
||||
# Set after recalculate_max_model_len to ensure that max_model_len
|
||||
# Set after get_and_verify_max_len to ensure that max_model_len
|
||||
# can be correctly capped to sliding window size
|
||||
self.hf_text_config.sliding_window = None
|
||||
|
||||
@@ -661,9 +715,10 @@ class ModelConfig:
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_model_config_after(self: "ModelConfig") -> "ModelConfig":
|
||||
if not isinstance(self.tokenizer, str):
|
||||
raise ValueError("tokenizer must be a string after __post_init__.")
|
||||
if not isinstance(self.max_model_len, int):
|
||||
raise ValueError("max_model_len must be an integer after __post_init__.")
|
||||
|
||||
return self
|
||||
|
||||
def _get_transformers_backend_cls(self) -> str:
|
||||
@@ -712,17 +767,49 @@ class ModelConfig:
|
||||
"""The architecture vllm actually used."""
|
||||
return self._architecture
|
||||
|
||||
def maybe_pull_model_for_runai(self, model: str) -> None:
|
||||
"""Pull model from Object Storage to temporary directory when needed."""
|
||||
if not is_runai_obj_uri(model):
|
||||
def maybe_pull_model_tokenizer_for_runai(self, model: str, tokenizer: str) -> None:
|
||||
"""Pull model/tokenizer from Object Storage to temporary
|
||||
directory when needed.
|
||||
|
||||
Args:
|
||||
model: Model name or path
|
||||
tokenizer: Tokenizer name or path
|
||||
"""
|
||||
|
||||
if not (is_runai_obj_uri(model) or is_runai_obj_uri(tokenizer)):
|
||||
return
|
||||
|
||||
object_storage_model = ObjectStorageModel(url=model)
|
||||
object_storage_model.pull_files(
|
||||
model, allow_pattern=["*.model", "*.py", "*.json"]
|
||||
)
|
||||
self.model_weights = model
|
||||
self.model = object_storage_model.dir
|
||||
if is_runai_obj_uri(model):
|
||||
object_storage_model = ObjectStorageModel(url=model)
|
||||
object_storage_model.pull_files(
|
||||
model, allow_pattern=["*.model", "*.py", "*.json"]
|
||||
)
|
||||
self.model_weights = model
|
||||
self.model = object_storage_model.dir
|
||||
|
||||
# If tokenizer is same as model, download to same directory
|
||||
if model == tokenizer:
|
||||
object_storage_model.pull_files(
|
||||
model,
|
||||
ignore_pattern=[
|
||||
"*.pt",
|
||||
"*.safetensors",
|
||||
"*.bin",
|
||||
"*.tensors",
|
||||
"*.pth",
|
||||
],
|
||||
)
|
||||
self.tokenizer = object_storage_model.dir
|
||||
return
|
||||
|
||||
# Only download tokenizer if needed and not already handled
|
||||
if is_runai_obj_uri(tokenizer):
|
||||
object_storage_tokenizer = ObjectStorageModel(url=tokenizer)
|
||||
object_storage_tokenizer.pull_files(
|
||||
model,
|
||||
ignore_pattern=["*.pt", "*.safetensors", "*.bin", "*.tensors", "*.pth"],
|
||||
)
|
||||
self.tokenizer = object_storage_tokenizer.dir
|
||||
|
||||
def _get_encoder_config(self):
|
||||
model = self.model
|
||||
@@ -1625,38 +1712,30 @@ class ModelConfig:
|
||||
return dense_modules[-1]["out_features"]
|
||||
return self.get_hidden_size()
|
||||
|
||||
def recalculate_max_model_len(
|
||||
self,
|
||||
original_max_model_len: int | None,
|
||||
*,
|
||||
tokenizer: str | None = None,
|
||||
tokenizer_revision: str | None = None,
|
||||
) -> None:
|
||||
def get_and_verify_max_len(self, max_model_len: int):
|
||||
# Consider max_model_len in tokenizer_config only when
|
||||
# pooling models use absolute position_embedding.
|
||||
# NOTE: For simplicity we assume `args.model == args.tokenizer`
|
||||
# since this is
|
||||
tokenizer_config = None
|
||||
if (
|
||||
self.runner_type == "pooling"
|
||||
and getattr(self.hf_config, "position_embedding_type", "") == "absolute"
|
||||
):
|
||||
tokenizer_config = try_get_tokenizer_config(
|
||||
tokenizer or self.model,
|
||||
self.tokenizer,
|
||||
trust_remote_code=self.trust_remote_code,
|
||||
revision=tokenizer_revision or self.revision,
|
||||
revision=self.tokenizer_revision,
|
||||
)
|
||||
|
||||
self.max_model_len = _get_and_verify_max_len(
|
||||
max_model_len = _get_and_verify_max_len(
|
||||
hf_config=self.hf_text_config,
|
||||
tokenizer_config=tokenizer_config,
|
||||
max_model_len=original_max_model_len,
|
||||
max_model_len=max_model_len,
|
||||
disable_sliding_window=self.disable_sliding_window,
|
||||
sliding_window=self.get_sliding_window(),
|
||||
spec_target_max_model_len=self.spec_target_max_model_len,
|
||||
encoder_config=self.encoder_config,
|
||||
)
|
||||
logger.info("Using max model len %s", self.max_model_len)
|
||||
logger.info("Using max model len %s", max_model_len)
|
||||
return max_model_len
|
||||
|
||||
@property
|
||||
def attn_type(self) -> AttnTypeStr:
|
||||
|
||||
@@ -79,6 +79,10 @@ class MultiModalConfig:
|
||||
|
||||
WARNING: The vLLM engine may crash if incorrect shape of embeddings is passed.
|
||||
Only enable this flag for trusted users!"""
|
||||
media_io_kwargs: dict[str, dict[str, Any]] = Field(default_factory=dict)
|
||||
"""Additional args passed to process media inputs, keyed by modalities.
|
||||
For example, to set num_frames for video, set
|
||||
`--media-io-kwargs '{"video": {"num_frames": 40} }'`"""
|
||||
mm_processor_kwargs: dict[str, object] | None = None
|
||||
"""Arguments to be forwarded to the model's processor for multi-modal data,
|
||||
e.g., image processor. Overrides for the multi-modal processor obtained
|
||||
|
||||
@@ -1,109 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import Field, SkipValidation
|
||||
from pydantic.dataclasses import dataclass
|
||||
|
||||
from vllm.config.model import ModelConfig
|
||||
from vllm.config.utils import config
|
||||
from vllm.transformers_utils.gguf_utils import is_gguf
|
||||
from vllm.transformers_utils.runai_utils import ObjectStorageModel, is_runai_obj_uri
|
||||
from vllm.transformers_utils.utils import maybe_model_redirect
|
||||
|
||||
TokenizerMode = Literal["auto", "hf", "slow", "mistral", "deepseek_v32"]
|
||||
|
||||
|
||||
@config
|
||||
@dataclass
|
||||
class RendererConfig:
|
||||
"""Configuration for the renderer."""
|
||||
|
||||
# NOTE: In reality, this is a required argument.
|
||||
# We provide a dummy default value here to generate the CLI args.
|
||||
model_config: SkipValidation[ModelConfig] = None # type: ignore
|
||||
"""Provides model context to the renderer."""
|
||||
|
||||
tokenizer: str = ""
|
||||
"""Name or path of the Hugging Face tokenizer to use. If unspecified, model
|
||||
name or path will be used."""
|
||||
tokenizer_mode: TokenizerMode | str = "auto"
|
||||
"""Tokenizer mode:\n
|
||||
- "auto" will use the tokenizer from `mistral_common` for Mistral models
|
||||
if available, otherwise it will use the "hf" tokenizer.\n
|
||||
- "hf" will use the fast tokenizer if available.\n
|
||||
- "slow" will always use the slow tokenizer.\n
|
||||
- "mistral" will always use the tokenizer from `mistral_common`.\n
|
||||
- "deepseek_v32" will always use the tokenizer from `deepseek_v32`.\n
|
||||
- Other custom values can be supported via plugins."""
|
||||
tokenizer_revision: str | None = None
|
||||
"""The specific revision to use for the tokenizer on the Hugging Face Hub.
|
||||
It can be a branch name, a tag name, or a commit id. If unspecified, will
|
||||
use the default version."""
|
||||
skip_tokenizer_init: bool = False
|
||||
"""Skip initialization of tokenizer and detokenizer. Expects valid
|
||||
`prompt_token_ids` and `None` for prompt from the input. The generated
|
||||
output will contain token ids."""
|
||||
|
||||
io_processor_plugin: str | None = None
|
||||
"""IOProcessor plugin name to load at model startup."""
|
||||
|
||||
media_io_kwargs: dict[str, dict[str, Any]] = Field(default_factory=dict)
|
||||
"""Additional args passed to process media inputs, keyed by modalities.
|
||||
For example, to set num_frames for video, set
|
||||
`--media-io-kwargs '{"video": {"num_frames": 40} }'`"""
|
||||
allowed_local_media_path: str = ""
|
||||
"""Allowing API requests to read local images or videos from directories
|
||||
specified by the server file system. This is a security risk. Should only
|
||||
be enabled in trusted environments."""
|
||||
allowed_media_domains: list[str] | None = None
|
||||
"""If set, only media URLs that belong to this domain can be used for
|
||||
multi-modal inputs. """
|
||||
|
||||
@property
|
||||
def trust_remote_code(self) -> bool:
|
||||
return self.model_config.trust_remote_code
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
model_config = self.model_config
|
||||
|
||||
# The tokenizer is consistent with the model by default.
|
||||
if not self.tokenizer:
|
||||
self.tokenizer = (
|
||||
ModelConfig.model
|
||||
if model_config is None
|
||||
else model_config.original_model
|
||||
)
|
||||
if not self.tokenizer_revision:
|
||||
self.tokenizer_revision = (
|
||||
ModelConfig.revision if model_config is None else model_config.revision
|
||||
)
|
||||
|
||||
self.original_tokenizer = self.tokenizer
|
||||
self.tokenizer = maybe_model_redirect(self.original_tokenizer)
|
||||
self.maybe_pull_tokenizer_for_runai(self.tokenizer)
|
||||
|
||||
# Multimodal GGUF models must use original repo for mm processing
|
||||
is_multimodal_model = (
|
||||
ModelConfig.is_multimodal_model
|
||||
if model_config is None
|
||||
else model_config.is_multimodal_model
|
||||
)
|
||||
if is_gguf(self.tokenizer) and is_multimodal_model:
|
||||
raise ValueError(
|
||||
"Loading a multimodal GGUF model needs to use original "
|
||||
"tokenizer. Please specify the unquantized hf model's "
|
||||
"repo name or path using the --tokenizer argument."
|
||||
)
|
||||
|
||||
def maybe_pull_tokenizer_for_runai(self, tokenizer: str) -> None:
|
||||
"""Pull tokenizer from Object Storage to temporary directory when needed."""
|
||||
if not is_runai_obj_uri(tokenizer):
|
||||
return
|
||||
|
||||
object_storage_tokenizer = ObjectStorageModel(url=tokenizer)
|
||||
object_storage_tokenizer.pull_files(
|
||||
tokenizer,
|
||||
ignore_pattern=["*.pt", "*.safetensors", "*.bin", "*.tensors", "*.pth"],
|
||||
)
|
||||
self.tokenizer = object_storage_tokenizer.dir
|
||||
@@ -322,11 +322,16 @@ class SpeculativeConfig:
|
||||
self.draft_model_config = ModelConfig(
|
||||
model=self.model,
|
||||
runner="draft",
|
||||
tokenizer=self.target_model_config.tokenizer,
|
||||
tokenizer_mode=self.target_model_config.tokenizer_mode,
|
||||
trust_remote_code=self.target_model_config.trust_remote_code,
|
||||
allowed_local_media_path=self.target_model_config.allowed_local_media_path,
|
||||
allowed_media_domains=self.target_model_config.allowed_media_domains,
|
||||
dtype=self.target_model_config.dtype,
|
||||
seed=self.target_model_config.seed,
|
||||
revision=self.revision,
|
||||
code_revision=self.code_revision,
|
||||
tokenizer_revision=self.target_model_config.tokenizer_revision,
|
||||
spec_target_max_model_len=self.target_model_config.max_model_len,
|
||||
quantization=self.quantization,
|
||||
enforce_eager=self.target_model_config.enforce_eager,
|
||||
|
||||
+10
-15
@@ -39,7 +39,6 @@ from .lora import LoRAConfig
|
||||
from .model import ModelConfig
|
||||
from .observability import ObservabilityConfig
|
||||
from .parallel import ParallelConfig
|
||||
from .renderer import RendererConfig
|
||||
from .scheduler import SchedulerConfig
|
||||
from .speculative import SpeculativeConfig
|
||||
from .structured_outputs import StructuredOutputsConfig
|
||||
@@ -182,8 +181,6 @@ class VllmConfig:
|
||||
# try to download a model
|
||||
model_config: ModelConfig = Field(default=None)
|
||||
"""Model configuration."""
|
||||
renderer_config: RendererConfig = Field(default_factory=RendererConfig)
|
||||
"""Renderer configuration."""
|
||||
cache_config: CacheConfig = Field(default_factory=CacheConfig)
|
||||
"""Cache configuration."""
|
||||
parallel_config: ParallelConfig = Field(default_factory=ParallelConfig)
|
||||
@@ -744,7 +741,7 @@ class VllmConfig:
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
self.scheduler_config.max_num_encoder_input_tokens = (
|
||||
MULTIMODAL_REGISTRY.get_encdec_max_encoder_len(self.renderer_config)
|
||||
MULTIMODAL_REGISTRY.get_encdec_max_encoder_len(self.model_config)
|
||||
)
|
||||
logger.debug(
|
||||
"Encoder-decoder model detected: setting "
|
||||
@@ -1189,13 +1186,11 @@ class VllmConfig:
|
||||
computed_compile_ranges_split_points
|
||||
)
|
||||
|
||||
def recalculate_max_model_len(self, original_max_model_len: int | None) -> None:
|
||||
# Can only be called during try_verify_and_update_config
|
||||
self.model_config.recalculate_max_model_len(
|
||||
original_max_model_len,
|
||||
tokenizer=self.renderer_config.tokenizer,
|
||||
tokenizer_revision=self.renderer_config.tokenizer_revision,
|
||||
)
|
||||
def recalculate_max_model_len(self, max_model_len: int):
|
||||
# Can only be called in try_verify_and_update_config
|
||||
model_config = self.model_config
|
||||
max_model_len = model_config.get_and_verify_max_len(max_model_len)
|
||||
self.model_config.max_model_len = max_model_len
|
||||
|
||||
def try_verify_and_update_config(self):
|
||||
if self.model_config is None:
|
||||
@@ -1269,11 +1264,11 @@ class VllmConfig:
|
||||
return (
|
||||
f"model={self.model_config.model!r}, "
|
||||
f"speculative_config={self.speculative_config!r}, "
|
||||
f"tokenizer={self.renderer_config.tokenizer!r}, "
|
||||
f"skip_tokenizer_init={self.renderer_config.skip_tokenizer_init}, "
|
||||
f"tokenizer_mode={self.renderer_config.tokenizer_mode}, "
|
||||
f"tokenizer={self.model_config.tokenizer!r}, "
|
||||
f"skip_tokenizer_init={self.model_config.skip_tokenizer_init}, "
|
||||
f"tokenizer_mode={self.model_config.tokenizer_mode}, "
|
||||
f"revision={self.model_config.revision}, "
|
||||
f"tokenizer_revision={self.renderer_config.tokenizer_revision}, "
|
||||
f"tokenizer_revision={self.model_config.tokenizer_revision}, "
|
||||
f"trust_remote_code={self.model_config.trust_remote_code}, "
|
||||
f"dtype={self.model_config.dtype}, "
|
||||
f"max_seq_len={self.model_config.max_model_len}, "
|
||||
|
||||
+40
-59
@@ -71,11 +71,11 @@ from vllm.config.model import (
|
||||
ModelDType,
|
||||
RunnerOption,
|
||||
TaskOption,
|
||||
TokenizerMode,
|
||||
)
|
||||
from vllm.config.multimodal import MMCacheType, MMEncoderTPMode
|
||||
from vllm.config.observability import DetailedTraceModules
|
||||
from vllm.config.parallel import DistributedExecutorBackend, ExpertPlacementStrategy
|
||||
from vllm.config.renderer import RendererConfig, TokenizerMode
|
||||
from vllm.config.scheduler import SchedulerPolicy
|
||||
from vllm.config.utils import get_field
|
||||
from vllm.config.vllm import OptimizationLevel
|
||||
@@ -355,12 +355,17 @@ class EngineArgs:
|
||||
|
||||
model: str = ModelConfig.model
|
||||
served_model_name: str | list[str] | None = ModelConfig.served_model_name
|
||||
tokenizer: str | None = ModelConfig.tokenizer
|
||||
hf_config_path: str | None = ModelConfig.hf_config_path
|
||||
runner: RunnerOption = ModelConfig.runner
|
||||
convert: ConvertOption = ModelConfig.convert
|
||||
task: TaskOption | None = ModelConfig.task
|
||||
skip_tokenizer_init: bool = ModelConfig.skip_tokenizer_init
|
||||
enable_prompt_embeds: bool = ModelConfig.enable_prompt_embeds
|
||||
tokenizer_mode: TokenizerMode | str = ModelConfig.tokenizer_mode
|
||||
trust_remote_code: bool = ModelConfig.trust_remote_code
|
||||
allowed_local_media_path: str = ModelConfig.allowed_local_media_path
|
||||
allowed_media_domains: list[str] | None = ModelConfig.allowed_media_domains
|
||||
download_dir: str | None = LoadConfig.download_dir
|
||||
safetensors_load_strategy: str = LoadConfig.safetensors_load_strategy
|
||||
load_format: str | LoadFormats = LoadConfig.load_format
|
||||
@@ -444,6 +449,7 @@ class EngineArgs:
|
||||
code_revision: str | None = ModelConfig.code_revision
|
||||
hf_token: bool | str | None = ModelConfig.hf_token
|
||||
hf_overrides: HfOverrides = get_field(ModelConfig, "hf_overrides")
|
||||
tokenizer_revision: str | None = ModelConfig.tokenizer_revision
|
||||
quantization: QuantizationMethods | None = ModelConfig.quantization
|
||||
enforce_eager: bool = ModelConfig.enforce_eager
|
||||
disable_custom_all_reduce: bool = ParallelConfig.disable_custom_all_reduce
|
||||
@@ -452,6 +458,9 @@ class EngineArgs:
|
||||
)
|
||||
enable_mm_embeds: bool = MultiModalConfig.enable_mm_embeds
|
||||
interleave_mm_strings: bool = MultiModalConfig.interleave_mm_strings
|
||||
media_io_kwargs: dict[str, dict[str, Any]] = get_field(
|
||||
MultiModalConfig, "media_io_kwargs"
|
||||
)
|
||||
mm_processor_kwargs: dict[str, Any] | None = MultiModalConfig.mm_processor_kwargs
|
||||
disable_mm_preprocessor_cache: bool = False # DEPRECATED
|
||||
mm_processor_cache_gb: float = MultiModalConfig.mm_processor_cache_gb
|
||||
@@ -465,19 +474,9 @@ class EngineArgs:
|
||||
mm_encoder_attn_backend: AttentionBackendEnum | str | None = (
|
||||
MultiModalConfig.mm_encoder_attn_backend
|
||||
)
|
||||
io_processor_plugin: str | None = None
|
||||
skip_mm_profiling: bool = MultiModalConfig.skip_mm_profiling
|
||||
video_pruning_rate: float = MultiModalConfig.video_pruning_rate
|
||||
# Renderer fields
|
||||
tokenizer: str | None = None
|
||||
tokenizer_mode: TokenizerMode | str = RendererConfig.tokenizer_mode
|
||||
tokenizer_revision: str | None = RendererConfig.tokenizer_revision
|
||||
skip_tokenizer_init: bool = RendererConfig.skip_tokenizer_init
|
||||
io_processor_plugin: str | None = None
|
||||
media_io_kwargs: dict[str, dict[str, Any]] = get_field(
|
||||
RendererConfig, "media_io_kwargs"
|
||||
)
|
||||
allowed_local_media_path: str = RendererConfig.allowed_local_media_path
|
||||
allowed_media_domains: list[str] | None = RendererConfig.allowed_media_domains
|
||||
# LoRA fields
|
||||
enable_lora: bool = False
|
||||
max_loras: int = LoRAConfig.max_loras
|
||||
@@ -628,14 +627,25 @@ class EngineArgs:
|
||||
model_group.add_argument("--runner", **model_kwargs["runner"])
|
||||
model_group.add_argument("--convert", **model_kwargs["convert"])
|
||||
model_group.add_argument("--task", **model_kwargs["task"], deprecated=True)
|
||||
model_group.add_argument("--tokenizer", **model_kwargs["tokenizer"])
|
||||
model_group.add_argument("--tokenizer-mode", **model_kwargs["tokenizer_mode"])
|
||||
model_group.add_argument(
|
||||
"--trust-remote-code", **model_kwargs["trust_remote_code"]
|
||||
)
|
||||
model_group.add_argument("--dtype", **model_kwargs["dtype"])
|
||||
model_group.add_argument("--seed", **model_kwargs["seed"])
|
||||
model_group.add_argument("--hf-config-path", **model_kwargs["hf_config_path"])
|
||||
model_group.add_argument(
|
||||
"--allowed-local-media-path", **model_kwargs["allowed_local_media_path"]
|
||||
)
|
||||
model_group.add_argument(
|
||||
"--allowed-media-domains", **model_kwargs["allowed_media_domains"]
|
||||
)
|
||||
model_group.add_argument("--revision", **model_kwargs["revision"])
|
||||
model_group.add_argument("--code-revision", **model_kwargs["code_revision"])
|
||||
model_group.add_argument(
|
||||
"--tokenizer-revision", **model_kwargs["tokenizer_revision"]
|
||||
)
|
||||
model_group.add_argument("--max-model-len", **model_kwargs["max_model_len"])
|
||||
model_group.add_argument("--quantization", "-q", **model_kwargs["quantization"])
|
||||
model_group.add_argument("--enforce-eager", **model_kwargs["enforce_eager"])
|
||||
@@ -647,6 +657,9 @@ class EngineArgs:
|
||||
model_group.add_argument(
|
||||
"--disable-cascade-attn", **model_kwargs["disable_cascade_attn"]
|
||||
)
|
||||
model_group.add_argument(
|
||||
"--skip-tokenizer-init", **model_kwargs["skip_tokenizer_init"]
|
||||
)
|
||||
model_group.add_argument(
|
||||
"--enable-prompt-embeds", **model_kwargs["enable_prompt_embeds"]
|
||||
)
|
||||
@@ -685,34 +698,8 @@ class EngineArgs:
|
||||
model_group.add_argument(
|
||||
"--logits-processors", **model_kwargs["logits_processors"]
|
||||
)
|
||||
|
||||
# Renderer arguments
|
||||
renderer_kwargs = get_kwargs(RendererConfig)
|
||||
renderer_group = parser.add_argument_group(
|
||||
title="RendererConfig",
|
||||
description=RendererConfig.__doc__,
|
||||
)
|
||||
renderer_group.add_argument("--tokenizer", **renderer_kwargs["tokenizer"])
|
||||
renderer_group.add_argument(
|
||||
"--tokenizer-mode", **renderer_kwargs["tokenizer_mode"]
|
||||
)
|
||||
renderer_group.add_argument(
|
||||
"--tokenizer-revision", **renderer_kwargs["tokenizer_revision"]
|
||||
)
|
||||
renderer_group.add_argument(
|
||||
"--skip-tokenizer-init", **renderer_kwargs["skip_tokenizer_init"]
|
||||
)
|
||||
renderer_group.add_argument(
|
||||
"--media-io-kwargs", **renderer_kwargs["media_io_kwargs"]
|
||||
)
|
||||
renderer_group.add_argument(
|
||||
"--allowed-local-media-path", **renderer_kwargs["allowed_local_media_path"]
|
||||
)
|
||||
renderer_group.add_argument(
|
||||
"--allowed-media-domains", **renderer_kwargs["allowed_media_domains"]
|
||||
)
|
||||
renderer_group.add_argument(
|
||||
"--io-processor-plugin", **renderer_kwargs["io_processor_plugin"]
|
||||
model_group.add_argument(
|
||||
"--io-processor-plugin", **model_kwargs["io_processor_plugin"]
|
||||
)
|
||||
|
||||
# Model loading arguments
|
||||
@@ -962,6 +949,9 @@ class EngineArgs:
|
||||
multimodal_group.add_argument(
|
||||
"--enable-mm-embeds", **multimodal_kwargs["enable_mm_embeds"]
|
||||
)
|
||||
multimodal_group.add_argument(
|
||||
"--media-io-kwargs", **multimodal_kwargs["media_io_kwargs"]
|
||||
)
|
||||
multimodal_group.add_argument(
|
||||
"--mm-processor-kwargs", **multimodal_kwargs["mm_processor_kwargs"]
|
||||
)
|
||||
@@ -1265,13 +1255,18 @@ class EngineArgs:
|
||||
runner=self.runner,
|
||||
convert=self.convert,
|
||||
task=self.task,
|
||||
tokenizer=self.tokenizer,
|
||||
tokenizer_mode=self.tokenizer_mode,
|
||||
trust_remote_code=self.trust_remote_code,
|
||||
allowed_local_media_path=self.allowed_local_media_path,
|
||||
allowed_media_domains=self.allowed_media_domains,
|
||||
dtype=self.dtype,
|
||||
seed=self.seed,
|
||||
revision=self.revision,
|
||||
code_revision=self.code_revision,
|
||||
hf_token=self.hf_token,
|
||||
hf_overrides=self.hf_overrides,
|
||||
tokenizer_revision=self.tokenizer_revision,
|
||||
max_model_len=self.max_model_len,
|
||||
quantization=self.quantization,
|
||||
enforce_eager=self.enforce_eager,
|
||||
@@ -1279,11 +1274,13 @@ class EngineArgs:
|
||||
logprobs_mode=self.logprobs_mode,
|
||||
disable_sliding_window=self.disable_sliding_window,
|
||||
disable_cascade_attn=self.disable_cascade_attn,
|
||||
skip_tokenizer_init=self.skip_tokenizer_init,
|
||||
enable_prompt_embeds=self.enable_prompt_embeds,
|
||||
served_model_name=self.served_model_name,
|
||||
limit_mm_per_prompt=self.limit_mm_per_prompt,
|
||||
enable_mm_embeds=self.enable_mm_embeds,
|
||||
interleave_mm_strings=self.interleave_mm_strings,
|
||||
media_io_kwargs=self.media_io_kwargs,
|
||||
skip_mm_profiling=self.skip_mm_profiling,
|
||||
config_format=self.config_format,
|
||||
mm_processor_kwargs=self.mm_processor_kwargs,
|
||||
@@ -1301,6 +1298,7 @@ class EngineArgs:
|
||||
override_attention_dtype=self.override_attention_dtype,
|
||||
logits_processors=self.logits_processors,
|
||||
video_pruning_rate=self.video_pruning_rate,
|
||||
io_processor_plugin=self.io_processor_plugin,
|
||||
)
|
||||
|
||||
def validate_tensorizer_args(self):
|
||||
@@ -1396,25 +1394,9 @@ class EngineArgs:
|
||||
)
|
||||
|
||||
model_config = self.create_model_config()
|
||||
renderer_config = RendererConfig(
|
||||
model_config=model_config,
|
||||
tokenizer=self.tokenizer or "",
|
||||
tokenizer_mode=self.tokenizer_mode,
|
||||
tokenizer_revision=self.tokenizer_revision,
|
||||
skip_tokenizer_init=self.skip_tokenizer_init,
|
||||
io_processor_plugin=self.io_processor_plugin,
|
||||
media_io_kwargs=self.media_io_kwargs,
|
||||
allowed_local_media_path=self.allowed_local_media_path,
|
||||
allowed_media_domains=self.allowed_media_domains,
|
||||
)
|
||||
|
||||
model_config.recalculate_max_model_len(
|
||||
model_config.original_max_model_len,
|
||||
tokenizer=renderer_config.tokenizer,
|
||||
tokenizer_revision=renderer_config.tokenizer_revision,
|
||||
)
|
||||
|
||||
self.model = model_config.model
|
||||
self.tokenizer = model_config.tokenizer
|
||||
|
||||
self._check_feature_supported(model_config)
|
||||
self._set_default_chunked_prefill_and_prefix_caching_args(model_config)
|
||||
self._set_default_max_num_seqs_and_batched_tokens_args(
|
||||
@@ -1786,7 +1768,6 @@ class EngineArgs:
|
||||
)
|
||||
config = VllmConfig(
|
||||
model_config=model_config,
|
||||
renderer_config=renderer_config,
|
||||
cache_config=cache_config,
|
||||
parallel_config=parallel_config,
|
||||
scheduler_config=scheduler_config,
|
||||
|
||||
@@ -5,7 +5,7 @@ from abc import ABC, abstractmethod
|
||||
from collections.abc import AsyncGenerator, Iterable, Mapping
|
||||
from typing import Any
|
||||
|
||||
from vllm.config import ModelConfig, RendererConfig, VllmConfig
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.inputs.data import PromptType
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.outputs import PoolingRequestOutput, RequestOutput
|
||||
@@ -22,7 +22,6 @@ class EngineClient(ABC):
|
||||
"""Protocol class for Clients to Engine"""
|
||||
|
||||
vllm_config: VllmConfig
|
||||
renderer_config: RendererConfig
|
||||
model_config: ModelConfig
|
||||
input_processor: InputProcessor
|
||||
io_processor: IOProcessor | None
|
||||
|
||||
@@ -44,7 +44,7 @@ from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast, Processor
|
||||
from typing_extensions import Required, TypedDict
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config import ModelConfig, RendererConfig
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models import SupportsMultiModal
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict, MultiModalUUIDDict
|
||||
@@ -452,10 +452,9 @@ This is needed because `lru_cache` does not cache when an exception happens.
|
||||
|
||||
def _try_get_processor_chat_template(
|
||||
tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast,
|
||||
*,
|
||||
trust_remote_code: bool,
|
||||
model_config: ModelConfig,
|
||||
) -> str | None:
|
||||
cache_key = (tokenizer.name_or_path, trust_remote_code)
|
||||
cache_key = (tokenizer.name_or_path, model_config.trust_remote_code)
|
||||
if cache_key in _PROCESSOR_CHAT_TEMPLATES:
|
||||
return _PROCESSOR_CHAT_TEMPLATES[cache_key]
|
||||
|
||||
@@ -467,7 +466,7 @@ def _try_get_processor_chat_template(
|
||||
PreTrainedTokenizerFast,
|
||||
ProcessorMixin,
|
||||
),
|
||||
trust_remote_code=trust_remote_code,
|
||||
trust_remote_code=model_config.trust_remote_code,
|
||||
)
|
||||
if (
|
||||
isinstance(processor, ProcessorMixin)
|
||||
@@ -500,10 +499,7 @@ def resolve_hf_chat_template(
|
||||
|
||||
# 2nd priority: AutoProcessor chat template, unless tool calling is enabled
|
||||
if tools is None:
|
||||
chat_template = _try_get_processor_chat_template(
|
||||
tokenizer,
|
||||
trust_remote_code=model_config.trust_remote_code,
|
||||
)
|
||||
chat_template = _try_get_processor_chat_template(tokenizer, model_config)
|
||||
if chat_template is not None:
|
||||
return chat_template
|
||||
|
||||
@@ -517,10 +513,10 @@ def resolve_hf_chat_template(
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
# 4th priority: Predefined fallbacks]
|
||||
# 4th priority: Predefined fallbacks
|
||||
path = get_chat_template_fallback_path(
|
||||
model_type=model_config.hf_config.model_type,
|
||||
tokenizer_name_or_path=tokenizer.name_or_path,
|
||||
tokenizer_name_or_path=model_config.tokenizer,
|
||||
)
|
||||
if path is not None:
|
||||
logger.info_once(
|
||||
@@ -542,14 +538,14 @@ def _resolve_chat_template_content_format(
|
||||
tools: list[dict[str, Any]] | None,
|
||||
tokenizer: TokenizerLike | None,
|
||||
*,
|
||||
renderer_config: RendererConfig,
|
||||
model_config: ModelConfig,
|
||||
) -> _ChatTemplateContentFormat:
|
||||
if isinstance(tokenizer, (PreTrainedTokenizer, PreTrainedTokenizerFast)):
|
||||
hf_chat_template = resolve_hf_chat_template(
|
||||
tokenizer,
|
||||
chat_template=chat_template,
|
||||
tools=tools,
|
||||
model_config=renderer_config.model_config,
|
||||
model_config=model_config,
|
||||
)
|
||||
else:
|
||||
hf_chat_template = None
|
||||
@@ -599,7 +595,7 @@ def resolve_chat_template_content_format(
|
||||
given_format: ChatTemplateContentFormatOption,
|
||||
tokenizer: TokenizerLike | None,
|
||||
*,
|
||||
renderer_config: RendererConfig,
|
||||
model_config: ModelConfig,
|
||||
) -> _ChatTemplateContentFormat:
|
||||
if given_format != "auto":
|
||||
return given_format
|
||||
@@ -608,7 +604,7 @@ def resolve_chat_template_content_format(
|
||||
chat_template,
|
||||
tools,
|
||||
tokenizer,
|
||||
renderer_config=renderer_config,
|
||||
model_config=model_config,
|
||||
)
|
||||
|
||||
_log_chat_template_content_format(
|
||||
@@ -631,32 +627,32 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
|
||||
maximum per prompt.
|
||||
"""
|
||||
|
||||
def __init__(self, renderer_config: RendererConfig):
|
||||
def __init__(self, model_config: ModelConfig):
|
||||
super().__init__()
|
||||
|
||||
self._renderer_config = renderer_config
|
||||
self._model_config = model_config
|
||||
|
||||
self._items_by_modality = defaultdict[str, list[_T | None]](list)
|
||||
self._uuids_by_modality = defaultdict[str, list[str | None]](list)
|
||||
|
||||
@property
|
||||
def renderer_config(self) -> RendererConfig:
|
||||
return self._renderer_config
|
||||
def model_config(self) -> ModelConfig:
|
||||
return self._model_config
|
||||
|
||||
@cached_property
|
||||
def model_cls(self) -> type[SupportsMultiModal]:
|
||||
from vllm.model_executor.model_loader import get_model_cls
|
||||
|
||||
model_cls = get_model_cls(self.renderer_config.model_config)
|
||||
model_cls = get_model_cls(self.model_config)
|
||||
return cast(type[SupportsMultiModal], model_cls)
|
||||
|
||||
@property
|
||||
def allowed_local_media_path(self):
|
||||
return self._renderer_config.allowed_local_media_path
|
||||
return self._model_config.allowed_local_media_path
|
||||
|
||||
@property
|
||||
def allowed_media_domains(self):
|
||||
return self._renderer_config.allowed_media_domains
|
||||
return self._model_config.allowed_media_domains
|
||||
|
||||
@property
|
||||
def mm_registry(self):
|
||||
@@ -664,7 +660,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
|
||||
|
||||
@cached_property
|
||||
def mm_processor(self):
|
||||
return self.mm_registry.create_processor(self.renderer_config)
|
||||
return self.mm_registry.create_processor(self.model_config)
|
||||
|
||||
def add(
|
||||
self,
|
||||
@@ -855,20 +851,19 @@ class MultiModalContentParser(BaseMultiModalContentParser):
|
||||
super().__init__()
|
||||
|
||||
self._tracker = tracker
|
||||
multimodal_config = self._tracker.model_config.multimodal_config
|
||||
media_io_kwargs = getattr(multimodal_config, "media_io_kwargs", None)
|
||||
|
||||
self._connector: MediaConnector = MEDIA_CONNECTOR_REGISTRY.load(
|
||||
envs.VLLM_MEDIA_CONNECTOR,
|
||||
media_io_kwargs=self.renderer_config.media_io_kwargs,
|
||||
media_io_kwargs=media_io_kwargs,
|
||||
allowed_local_media_path=tracker.allowed_local_media_path,
|
||||
allowed_media_domains=tracker.allowed_media_domains,
|
||||
)
|
||||
|
||||
@property
|
||||
def renderer_config(self) -> RendererConfig:
|
||||
return self._tracker.renderer_config
|
||||
|
||||
@property
|
||||
def model_config(self) -> ModelConfig:
|
||||
return self.renderer_config.model_config
|
||||
return self._tracker.model_config
|
||||
|
||||
def parse_image(self, image_url: str | None, uuid: str | None = None) -> None:
|
||||
image = self._connector.fetch_image(image_url) if image_url else None
|
||||
@@ -968,20 +963,18 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
|
||||
super().__init__()
|
||||
|
||||
self._tracker = tracker
|
||||
multimodal_config = self._tracker.model_config.multimodal_config
|
||||
media_io_kwargs = getattr(multimodal_config, "media_io_kwargs", None)
|
||||
self._connector: MediaConnector = MEDIA_CONNECTOR_REGISTRY.load(
|
||||
envs.VLLM_MEDIA_CONNECTOR,
|
||||
media_io_kwargs=self.renderer_config.media_io_kwargs,
|
||||
media_io_kwargs=media_io_kwargs,
|
||||
allowed_local_media_path=tracker.allowed_local_media_path,
|
||||
allowed_media_domains=tracker.allowed_media_domains,
|
||||
)
|
||||
|
||||
@property
|
||||
def renderer_config(self) -> RendererConfig:
|
||||
return self._tracker.renderer_config
|
||||
|
||||
@property
|
||||
def model_config(self) -> ModelConfig:
|
||||
return self.renderer_config.model_config
|
||||
return self._tracker.model_config
|
||||
|
||||
def parse_image(self, image_url: str | None, uuid: str | None = None) -> None:
|
||||
image_coro = self._connector.fetch_image_async(image_url) if image_url else None
|
||||
@@ -1611,17 +1604,15 @@ def _postprocess_messages(messages: list[ConversationMessage]) -> None:
|
||||
|
||||
def parse_chat_messages(
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
renderer_config: RendererConfig,
|
||||
model_config: ModelConfig,
|
||||
content_format: _ChatTemplateContentFormat,
|
||||
) -> tuple[
|
||||
list[ConversationMessage],
|
||||
MultiModalDataDict | None,
|
||||
MultiModalUUIDDict | None,
|
||||
]:
|
||||
model_config = renderer_config.model_config
|
||||
|
||||
conversation: list[ConversationMessage] = []
|
||||
mm_tracker = MultiModalItemTracker(renderer_config)
|
||||
mm_tracker = MultiModalItemTracker(model_config)
|
||||
|
||||
for msg in messages:
|
||||
sub_messages = _parse_chat_message_content(
|
||||
@@ -1644,17 +1635,15 @@ def parse_chat_messages(
|
||||
|
||||
def parse_chat_messages_futures(
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
renderer_config: RendererConfig,
|
||||
model_config: ModelConfig,
|
||||
content_format: _ChatTemplateContentFormat,
|
||||
) -> tuple[
|
||||
list[ConversationMessage],
|
||||
Awaitable[MultiModalDataDict | None],
|
||||
MultiModalUUIDDict | None,
|
||||
]:
|
||||
model_config = renderer_config.model_config
|
||||
|
||||
conversation: list[ConversationMessage] = []
|
||||
mm_tracker = AsyncMultiModalItemTracker(renderer_config)
|
||||
mm_tracker = AsyncMultiModalItemTracker(model_config)
|
||||
|
||||
for msg in messages:
|
||||
sub_messages = _parse_chat_message_content(
|
||||
@@ -1759,14 +1748,14 @@ def apply_hf_chat_template(
|
||||
chat_template: str | None,
|
||||
tools: list[dict[str, Any]] | None,
|
||||
*,
|
||||
renderer_config: RendererConfig,
|
||||
model_config: ModelConfig,
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
hf_chat_template = resolve_hf_chat_template(
|
||||
tokenizer,
|
||||
chat_template=chat_template,
|
||||
tools=tools,
|
||||
model_config=renderer_config.model_config,
|
||||
model_config=model_config,
|
||||
)
|
||||
|
||||
if hf_chat_template is None:
|
||||
|
||||
@@ -29,8 +29,8 @@ from vllm.config.model import (
|
||||
HfOverrides,
|
||||
ModelDType,
|
||||
RunnerOption,
|
||||
TokenizerMode,
|
||||
)
|
||||
from vllm.config.renderer import TokenizerMode
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
@@ -343,7 +343,6 @@ class LLM:
|
||||
logger.info("Supported tasks: %s", supported_tasks)
|
||||
self.supported_tasks = supported_tasks
|
||||
|
||||
self.renderer_config = self.llm_engine.renderer_config
|
||||
self.model_config = self.llm_engine.model_config
|
||||
self.input_processor = self.llm_engine.input_processor
|
||||
self.io_processor = self.llm_engine.io_processor
|
||||
@@ -809,13 +808,13 @@ class LLM:
|
||||
list_of_messages = [cast(list[ChatCompletionMessageParam], messages)]
|
||||
|
||||
tokenizer = self.get_tokenizer()
|
||||
renderer_config = self.renderer_config
|
||||
model_config = self.model_config
|
||||
resolved_content_format = resolve_chat_template_content_format(
|
||||
chat_template,
|
||||
tools,
|
||||
chat_template_content_format,
|
||||
tokenizer,
|
||||
renderer_config=renderer_config,
|
||||
model_config=model_config,
|
||||
)
|
||||
|
||||
_chat_template_kwargs: dict[str, Any] = dict(
|
||||
@@ -834,7 +833,7 @@ class LLM:
|
||||
# the chat message parsing for it.
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
msgs,
|
||||
renderer_config,
|
||||
model_config,
|
||||
content_format=resolved_content_format,
|
||||
)
|
||||
|
||||
@@ -848,7 +847,7 @@ class LLM:
|
||||
prompt_str = apply_hf_chat_template(
|
||||
tokenizer=tokenizer,
|
||||
conversation=conversation,
|
||||
renderer_config=renderer_config,
|
||||
model_config=model_config,
|
||||
**_chat_template_kwargs,
|
||||
)
|
||||
# Special tokens are already included in chat templates so
|
||||
@@ -1291,7 +1290,6 @@ class LLM:
|
||||
lora_request: list[LoRARequest] | LoRARequest | None = None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
) -> list[ScoringRequestOutput]:
|
||||
renderer_config = self.renderer_config
|
||||
model_config = self.model_config
|
||||
|
||||
if isinstance(tokenizer, MistralTokenizer):
|
||||
@@ -1319,7 +1317,7 @@ class LLM:
|
||||
|
||||
for q, d in input_pairs:
|
||||
_, engine_prompt = get_score_prompt(
|
||||
renderer_config=renderer_config,
|
||||
model_config=model_config,
|
||||
data_1=q,
|
||||
data_2=d,
|
||||
tokenizer=tokenizer,
|
||||
|
||||
@@ -1099,7 +1099,7 @@ async def init_app_state(
|
||||
logger.info("Supported tasks: %s", supported_tasks)
|
||||
|
||||
resolved_chat_template = await process_chat_template(
|
||||
args.chat_template, engine_client, vllm_config.renderer_config
|
||||
args.chat_template, engine_client, vllm_config.model_config
|
||||
)
|
||||
|
||||
if args.tool_server == "demo":
|
||||
|
||||
@@ -122,7 +122,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
try:
|
||||
lora_request = self._maybe_get_adapters(request)
|
||||
|
||||
if self.renderer_config.skip_tokenizer_init:
|
||||
if self.model_config.skip_tokenizer_init:
|
||||
tokenizer = None
|
||||
else:
|
||||
tokenizer = await self.engine_client.get_tokenizer()
|
||||
|
||||
@@ -291,7 +291,6 @@ class OpenAIServing:
|
||||
|
||||
self.input_processor = self.models.input_processor
|
||||
self.io_processor = self.models.io_processor
|
||||
self.renderer_config = self.models.renderer_config
|
||||
self.model_config = self.models.model_config
|
||||
self.max_model_len = self.model_config.max_model_len
|
||||
|
||||
@@ -1101,18 +1100,18 @@ class OpenAIServing:
|
||||
Sequence[RequestPrompt],
|
||||
list[EngineTokensPrompt],
|
||||
]:
|
||||
renderer_config = self.renderer_config
|
||||
model_config = self.model_config
|
||||
|
||||
resolved_content_format = resolve_chat_template_content_format(
|
||||
chat_template,
|
||||
tool_dicts,
|
||||
chat_template_content_format,
|
||||
tokenizer,
|
||||
renderer_config=renderer_config,
|
||||
model_config=model_config,
|
||||
)
|
||||
conversation, mm_data_future, mm_uuids = parse_chat_messages_futures(
|
||||
messages,
|
||||
renderer_config,
|
||||
model_config,
|
||||
content_format=resolved_content_format,
|
||||
)
|
||||
|
||||
@@ -1139,14 +1138,14 @@ class OpenAIServing:
|
||||
request_prompt = tokenizer.apply_chat_template(
|
||||
conversation=conversation,
|
||||
messages=messages,
|
||||
model_config=renderer_config.model_config,
|
||||
model_config=model_config,
|
||||
**_chat_template_kwargs,
|
||||
)
|
||||
else:
|
||||
request_prompt = apply_hf_chat_template(
|
||||
tokenizer=tokenizer,
|
||||
conversation=conversation,
|
||||
renderer_config=renderer_config,
|
||||
model_config=model_config,
|
||||
**_chat_template_kwargs,
|
||||
)
|
||||
|
||||
|
||||
@@ -71,7 +71,6 @@ class OpenAIServingModels:
|
||||
|
||||
self.input_processor = self.engine_client.input_processor
|
||||
self.io_processor = self.engine_client.io_processor
|
||||
self.renderer_config = self.engine_client.renderer_config
|
||||
self.model_config = self.engine_client.model_config
|
||||
self.max_model_len = self.model_config.max_model_len
|
||||
|
||||
|
||||
@@ -91,7 +91,7 @@ class OpenAISpeechToText(OpenAIServing):
|
||||
self.task_type = task_type
|
||||
|
||||
self.asr_config = self.model_cls.get_speech_to_text_config(
|
||||
self.renderer_config, task_type
|
||||
self.model_config, task_type
|
||||
)
|
||||
|
||||
self.enable_force_include_usage = enable_force_include_usage
|
||||
@@ -101,8 +101,8 @@ class OpenAISpeechToText(OpenAIServing):
|
||||
self.tokenizer = cast(
|
||||
PreTrainedTokenizerBase,
|
||||
get_tokenizer(
|
||||
tokenizer_name=self.renderer_config.tokenizer,
|
||||
tokenizer_mode=self.renderer_config.tokenizer_mode,
|
||||
tokenizer_name=self.model_config.tokenizer,
|
||||
tokenizer_mode=self.model_config.tokenizer_mode,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -154,7 +154,7 @@ class OpenAISpeechToText(OpenAIServing):
|
||||
prompt = self.model_cls.get_generation_prompt(
|
||||
audio=chunk,
|
||||
stt_config=self.asr_config,
|
||||
renderer_config=self.renderer_config,
|
||||
model_config=self.model_config,
|
||||
language=language,
|
||||
task_type=self.task_type,
|
||||
request_prompt=request.prompt,
|
||||
@@ -428,7 +428,7 @@ class OpenAISpeechToText(OpenAIServing):
|
||||
if res.prompt_token_ids is not None:
|
||||
num_prompt_tokens = len(res.prompt_token_ids)
|
||||
if audio_tokens := self.model_cls.get_num_audio_tokens(
|
||||
audio_duration_s, self.asr_config, self.renderer_config
|
||||
audio_duration_s, self.asr_config, self.model_config
|
||||
):
|
||||
num_prompt_tokens += audio_tokens
|
||||
|
||||
|
||||
@@ -94,7 +94,7 @@ class OpenAIServingPooling(OpenAIServing):
|
||||
try:
|
||||
lora_request = self._maybe_get_adapters(request)
|
||||
|
||||
if self.renderer_config.skip_tokenizer_init:
|
||||
if self.model_config.skip_tokenizer_init:
|
||||
tokenizer = None
|
||||
else:
|
||||
tokenizer = await self.engine_client.get_tokenizer()
|
||||
|
||||
@@ -160,8 +160,10 @@ class ServingScores(OpenAIServing):
|
||||
data_1: str | ScoreContentPartParam,
|
||||
data_2: str | ScoreContentPartParam,
|
||||
) -> tuple[str, TokensPrompt]:
|
||||
model_config = self.model_config
|
||||
|
||||
full_prompt, engine_prompt = get_score_prompt(
|
||||
renderer_config=self.renderer_config,
|
||||
model_config=model_config,
|
||||
data_1=data_1,
|
||||
data_2=data_2,
|
||||
tokenizer=tokenizer,
|
||||
|
||||
@@ -5,7 +5,7 @@ from typing import Any, TypeAlias, cast
|
||||
from torch.nn import CosineSimilarity
|
||||
from typing_extensions import Required, TypedDict
|
||||
|
||||
from vllm.config import ModelConfig, RendererConfig
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
BaseMultiModalItemTracker,
|
||||
ChatCompletionContentPartImageEmbedsParam,
|
||||
@@ -88,9 +88,9 @@ def _validate_score_input_lens(
|
||||
def parse_score_data(
|
||||
data_1: str | ScoreContentPartParam,
|
||||
data_2: str | ScoreContentPartParam,
|
||||
renderer_config: RendererConfig,
|
||||
model_config: ModelConfig,
|
||||
) -> tuple[str, str, MultiModalDataDict | None]:
|
||||
mm_tracker = MultiModalItemTracker(renderer_config)
|
||||
mm_tracker = MultiModalItemTracker(model_config)
|
||||
|
||||
content_1 = _parse_score_content(data_1, mm_tracker)
|
||||
content_2 = _parse_score_content(data_2, mm_tracker)
|
||||
@@ -176,7 +176,7 @@ def post_process_tokens(
|
||||
|
||||
|
||||
def get_score_prompt(
|
||||
renderer_config: RendererConfig,
|
||||
model_config: ModelConfig,
|
||||
tokenizer: TokenizerLike,
|
||||
tokenization_kwargs: dict[str, Any],
|
||||
data_1: str | ScoreContentPartParam,
|
||||
@@ -185,14 +185,11 @@ def get_score_prompt(
|
||||
prompt_1, prompt_2, mm_data = parse_score_data(
|
||||
data_1,
|
||||
data_2,
|
||||
renderer_config,
|
||||
model_config,
|
||||
)
|
||||
|
||||
from vllm.model_executor.model_loader import get_model_cls
|
||||
|
||||
model_config = renderer_config.model_config
|
||||
model = get_model_cls(model_config)
|
||||
|
||||
if supports_score_template(model):
|
||||
full_prompt = apply_score_template(model_config, prompt_1, prompt_2)
|
||||
prompt_inputs = tokenizer(full_prompt, **tokenization_kwargs)
|
||||
|
||||
@@ -13,7 +13,7 @@ from fastapi import Request
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
from starlette.background import BackgroundTask, BackgroundTasks
|
||||
|
||||
from vllm.config import RendererConfig
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
@@ -288,7 +288,7 @@ def process_lora_modules(
|
||||
async def process_chat_template(
|
||||
args_chat_template: Path | str | None,
|
||||
engine_client: EngineClient,
|
||||
renderer_config: RendererConfig,
|
||||
model_config: ModelConfig,
|
||||
) -> str | None:
|
||||
resolved_chat_template = load_chat_template(args_chat_template)
|
||||
if resolved_chat_template is not None:
|
||||
@@ -305,7 +305,7 @@ async def process_chat_template(
|
||||
tokenizer=tokenizer,
|
||||
chat_template=None,
|
||||
tools=None,
|
||||
model_config=renderer_config.model_config,
|
||||
model_config=model_config,
|
||||
)
|
||||
|
||||
if hf_chat_template != resolved_chat_template:
|
||||
@@ -314,6 +314,6 @@ async def process_chat_template(
|
||||
"It is different from official chat template '%s'. "
|
||||
"This discrepancy may lead to performance degradation.",
|
||||
resolved_chat_template,
|
||||
renderer_config.model_config.model,
|
||||
model_config.model,
|
||||
)
|
||||
return resolved_chat_template
|
||||
|
||||
@@ -6,7 +6,7 @@ from typing import Any, cast
|
||||
|
||||
from typing_extensions import assert_never
|
||||
|
||||
from vllm.config import RendererConfig
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
|
||||
from vllm.multimodal.cache import BaseMultiModalProcessorCache
|
||||
@@ -45,15 +45,14 @@ logger = init_logger(__name__)
|
||||
class InputPreprocessor:
|
||||
def __init__(
|
||||
self,
|
||||
renderer_config: RendererConfig,
|
||||
model_config: ModelConfig,
|
||||
tokenizer: TokenizerLike | None,
|
||||
mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
|
||||
mm_processor_cache: BaseMultiModalProcessorCache | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.renderer_config = renderer_config
|
||||
self.model_config = renderer_config.model_config
|
||||
self.model_config = model_config
|
||||
self.tokenizer = tokenizer
|
||||
self.mm_registry = mm_registry
|
||||
self.mm_processor_cache = mm_processor_cache
|
||||
@@ -232,7 +231,7 @@ class InputPreprocessor:
|
||||
def _get_mm_processor(self) -> BaseMultiModalProcessor:
|
||||
if not hasattr(self, "_mm_processor"):
|
||||
self._mm_processor = self.mm_registry.create_processor(
|
||||
self.renderer_config,
|
||||
self.model_config,
|
||||
tokenizer=self.tokenizer,
|
||||
cache=self.mm_processor_cache,
|
||||
)
|
||||
|
||||
@@ -415,7 +415,7 @@ def load_weights_using_from_2_way_softmax(
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
renderer_config = model.vllm_config.renderer_config
|
||||
model_config = model.vllm_config.model_config
|
||||
quant_config = model.vllm_config.quant_config
|
||||
text_config = model.config.get_text_config()
|
||||
|
||||
@@ -447,10 +447,10 @@ def load_weights_using_from_2_way_softmax(
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
|
||||
tokenizer = get_tokenizer(
|
||||
renderer_config.tokenizer,
|
||||
revision=renderer_config.tokenizer_revision,
|
||||
tokenizer_mode=renderer_config.tokenizer_mode,
|
||||
trust_remote_code=renderer_config.trust_remote_code,
|
||||
model_config.tokenizer,
|
||||
revision=model_config.tokenizer_revision,
|
||||
tokenizer_mode=model_config.tokenizer_mode,
|
||||
trust_remote_code=model_config.trust_remote_code,
|
||||
)
|
||||
|
||||
false_id = tokenizer.convert_tokens_to_ids(tokens[0])
|
||||
@@ -473,7 +473,7 @@ def load_weights_no_post_processing(model, weights: Iterable[tuple[str, torch.Te
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
renderer_config = model.vllm_config.renderer_config
|
||||
model_config = model.vllm_config.model_config
|
||||
quant_config = model.vllm_config.quant_config
|
||||
text_config = model.config.get_text_config()
|
||||
|
||||
@@ -501,10 +501,10 @@ def load_weights_no_post_processing(model, weights: Iterable[tuple[str, torch.Te
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
|
||||
tokenizer = get_tokenizer(
|
||||
renderer_config.tokenizer,
|
||||
revision=renderer_config.tokenizer_revision,
|
||||
tokenizer_mode=renderer_config.tokenizer_mode,
|
||||
trust_remote_code=renderer_config.trust_remote_code,
|
||||
model_config.tokenizer,
|
||||
revision=model_config.tokenizer_revision,
|
||||
tokenizer_mode=model_config.tokenizer_mode,
|
||||
trust_remote_code=model_config.trust_remote_code,
|
||||
)
|
||||
|
||||
token_ids = [tokenizer.convert_tokens_to_ids(t) for t in tokens]
|
||||
|
||||
@@ -377,8 +377,8 @@ class DeepseekOCRForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
self.projector_config = config.projector_config
|
||||
self.text_config = config.text_config
|
||||
|
||||
renderer_config = vllm_config.renderer_config
|
||||
tokenizer = cached_tokenizer_from_config(renderer_config)
|
||||
model_config = vllm_config.model_config
|
||||
tokenizer = cached_tokenizer_from_config(model_config)
|
||||
self.image_token_id = tokenizer.vocab[_IMAGE_TOKEN]
|
||||
|
||||
self.sam_model = build_sam_vit_b()
|
||||
|
||||
@@ -370,8 +370,8 @@ class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
self.projector_config = config.projector_config
|
||||
self.text_config = config.text_config
|
||||
|
||||
renderer_config = vllm_config.renderer_config
|
||||
tokenizer = cached_tokenizer_from_config(renderer_config)
|
||||
model_config = vllm_config.model_config
|
||||
tokenizer = cached_tokenizer_from_config(model_config)
|
||||
self.image_token_id: int = tokenizer.vocab[_IMAGE_TOKEN]
|
||||
|
||||
self.vision = self._init_vision_module(
|
||||
|
||||
@@ -18,7 +18,7 @@ from transformers.models.gemma3n import (
|
||||
)
|
||||
from transformers.models.siglip import SiglipImageProcessorFast
|
||||
|
||||
from vllm.config import RendererConfig, SpeechToTextConfig, VllmConfig
|
||||
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
|
||||
from vllm.config.multimodal import BaseDummyOptions
|
||||
from vllm.inputs.data import PromptType
|
||||
from vllm.logger import init_logger
|
||||
@@ -760,7 +760,7 @@ class Gemma3nForConditionalGeneration(
|
||||
cls,
|
||||
audio: np.ndarray,
|
||||
stt_config: SpeechToTextConfig,
|
||||
renderer_config: RendererConfig,
|
||||
model_config: ModelConfig,
|
||||
language: Optional[str],
|
||||
task_type: Literal["transcribe", "translate"],
|
||||
request_prompt: str,
|
||||
@@ -798,9 +798,7 @@ class Gemma3nForConditionalGeneration(
|
||||
|
||||
@classmethod
|
||||
def get_speech_to_text_config(
|
||||
cls,
|
||||
renderer_config: RendererConfig,
|
||||
task_type: str,
|
||||
cls, model_config: ModelConfig, task_type: str
|
||||
) -> SpeechToTextConfig:
|
||||
return SpeechToTextConfig(
|
||||
# Let's set this to 30 as suggested in the docs for now, although
|
||||
|
||||
@@ -34,7 +34,7 @@ import torch.nn.functional as F
|
||||
from torch import nn
|
||||
from transformers import BatchFeature, PretrainedConfig
|
||||
|
||||
from vllm.config import CacheConfig, RendererConfig, SpeechToTextConfig, VllmConfig
|
||||
from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
|
||||
from vllm.config.multimodal import BaseDummyOptions
|
||||
from vllm.inputs.data import PromptType
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||
@@ -840,7 +840,7 @@ class GraniteSpeechForConditionalGeneration(
|
||||
def get_generation_prompt(
|
||||
cls,
|
||||
audio: np.ndarray,
|
||||
renderer_config: RendererConfig,
|
||||
model_config: ModelConfig,
|
||||
stt_config: SpeechToTextConfig,
|
||||
language: str | None,
|
||||
task_type: Literal["transcribe", "translate"],
|
||||
@@ -861,7 +861,7 @@ class GraniteSpeechForConditionalGeneration(
|
||||
else:
|
||||
raise ValueError(f"Unsupported task type {task_type}")
|
||||
|
||||
tokenizer = cached_tokenizer_from_config(renderer_config)
|
||||
tokenizer = cached_tokenizer_from_config(model_config)
|
||||
chat = [dict(role="user", content=user_prompt)]
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
chat,
|
||||
@@ -882,10 +882,10 @@ class GraniteSpeechForConditionalGeneration(
|
||||
cls,
|
||||
audio_duration_s: float,
|
||||
stt_config: SpeechToTextConfig,
|
||||
renderer_config: RendererConfig,
|
||||
model_config: ModelConfig,
|
||||
) -> int | None:
|
||||
"""Get the number of audio tokens for an audio duration in sec."""
|
||||
processor = cached_processor_from_config(renderer_config)
|
||||
processor = cached_processor_from_config(model_config)
|
||||
hop_length = processor.audio_processor.melspec_kwargs["hop_length"]
|
||||
proj_win_size = processor.audio_processor.projector_window_size
|
||||
ds_rate = processor.audio_processor.projector_downsample_rate
|
||||
@@ -903,9 +903,7 @@ class GraniteSpeechForConditionalGeneration(
|
||||
|
||||
@classmethod
|
||||
def get_speech_to_text_config(
|
||||
cls,
|
||||
renderer_config: RendererConfig,
|
||||
task_type: str,
|
||||
cls, model_config: ModelConfig, task_type: str
|
||||
) -> SpeechToTextConfig:
|
||||
"""Get the stt config for this model."""
|
||||
# Default settings are reasonable for this model and we don't currently
|
||||
|
||||
@@ -6,7 +6,7 @@ import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import RendererConfig, VllmConfig
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.pooler import (
|
||||
DispatchPooler,
|
||||
@@ -29,12 +29,12 @@ logger = init_logger(__name__)
|
||||
class GritLMMeanPool(nn.Module):
|
||||
"""As `MeanPool`, but only includes non-instruction tokens."""
|
||||
|
||||
def __init__(self, renderer_config: RendererConfig):
|
||||
def __init__(self, model_config: ModelConfig):
|
||||
super().__init__()
|
||||
|
||||
self.renderer_config = renderer_config
|
||||
self.model_config = model_config
|
||||
|
||||
tokenizer = cached_tokenizer_from_config(self.renderer_config)
|
||||
tokenizer = cached_tokenizer_from_config(self.model_config)
|
||||
|
||||
# Collect the tokens needed for pattern matching.
|
||||
# "▁<" is different from "_<". The former uses "▁" to indicate that
|
||||
@@ -174,10 +174,10 @@ class GritLMMeanPool(nn.Module):
|
||||
|
||||
|
||||
class GritLMPooler(Pooler):
|
||||
def __init__(self, renderer_config: RendererConfig):
|
||||
def __init__(self, model_config: ModelConfig):
|
||||
super().__init__()
|
||||
|
||||
self.pooling = GritLMMeanPool(renderer_config)
|
||||
self.pooling = GritLMMeanPool(model_config)
|
||||
self.head = PoolerHead(PoolerNormalize())
|
||||
|
||||
def get_supported_tasks(self) -> Set[PoolingTask]:
|
||||
@@ -238,6 +238,6 @@ class GritLM(LlamaForCausalLM):
|
||||
self.pooler = DispatchPooler(
|
||||
{
|
||||
"token_embed": Pooler.for_token_embed(pooler_config),
|
||||
"embed": GritLMPooler(vllm_config.renderer_config),
|
||||
"embed": GritLMPooler(vllm_config.model_config),
|
||||
}
|
||||
)
|
||||
|
||||
@@ -19,7 +19,7 @@ from torch import Tensor
|
||||
from transformers.models.whisper.tokenization_whisper import LANGUAGES
|
||||
from typing_extensions import Self, TypeIs
|
||||
|
||||
from vllm.config import RendererConfig, SpeechToTextConfig
|
||||
from vllm.config import ModelConfig, SpeechToTextConfig
|
||||
from vllm.inputs import TokensPrompt
|
||||
from vllm.inputs.data import PromptType
|
||||
from vllm.logger import init_logger
|
||||
@@ -887,7 +887,7 @@ class SupportsTranscription(Protocol):
|
||||
cls,
|
||||
audio: np.ndarray,
|
||||
stt_config: SpeechToTextConfig,
|
||||
renderer_config: RendererConfig,
|
||||
model_config: ModelConfig,
|
||||
language: str | None,
|
||||
task_type: Literal["transcribe", "translate"],
|
||||
request_prompt: str,
|
||||
@@ -930,9 +930,7 @@ class SupportsTranscription(Protocol):
|
||||
|
||||
@classmethod
|
||||
def get_speech_to_text_config(
|
||||
cls,
|
||||
renderer_config: RendererConfig,
|
||||
task_type: Literal["transcribe", "translate"],
|
||||
cls, model_config: ModelConfig, task_type: Literal["transcribe", "translate"]
|
||||
) -> SpeechToTextConfig:
|
||||
"""Get the speech to text config for the ASR model."""
|
||||
...
|
||||
@@ -942,7 +940,7 @@ class SupportsTranscription(Protocol):
|
||||
cls,
|
||||
audio_duration_s: float,
|
||||
stt_config: SpeechToTextConfig,
|
||||
renderer_config: RendererConfig,
|
||||
model_config: ModelConfig,
|
||||
) -> int | None:
|
||||
"""
|
||||
Map from audio duration to number of audio tokens produced by the ASR
|
||||
|
||||
@@ -182,7 +182,7 @@ class InternS1ProcessingInfo(BaseProcessingInfo):
|
||||
def get_hf_processor(self, **kwargs: object) -> InternVLProcessor:
|
||||
hf_processor = self.ctx.get_hf_processor(InternVLProcessor, **kwargs)
|
||||
hf_processor.video_processor = cached_video_processor_from_config(
|
||||
self.ctx.renderer_config,
|
||||
self.ctx.model_config,
|
||||
processor_cls=InternVLVideoProcessor,
|
||||
size=hf_processor.image_processor.size,
|
||||
**kwargs,
|
||||
|
||||
@@ -1169,17 +1169,16 @@ class NemotronH_Nano_VL_V2(
|
||||
self.mlp1 = self.mlp1.to(self.language_model.config.dtype)
|
||||
|
||||
self.config = config
|
||||
self.model_config = vllm_config.model_config
|
||||
|
||||
# Pre-tokenize special tokens for video processing
|
||||
# to avoid repeated tokenization
|
||||
self._tokenizer = cached_tokenizer_from_config(vllm_config.renderer_config)
|
||||
self._img_start_token_ids = self._tokenizer.encode(
|
||||
tokenizer = cached_tokenizer_from_config(vllm_config.model_config)
|
||||
self._img_start_token_ids = tokenizer.encode(
|
||||
IMG_START, add_special_tokens=False
|
||||
)
|
||||
self._img_end_token_ids = self._tokenizer.encode(
|
||||
IMG_END, add_special_tokens=False
|
||||
)
|
||||
self._img_context_token_ids = self._tokenizer.encode(
|
||||
self._img_end_token_ids = tokenizer.encode(IMG_END, add_special_tokens=False)
|
||||
self._img_context_token_ids = tokenizer.encode(
|
||||
IMG_CONTEXT, add_special_tokens=False
|
||||
)
|
||||
|
||||
@@ -1365,7 +1364,7 @@ class NemotronH_Nano_VL_V2(
|
||||
input_embeds for the LLM.
|
||||
"""
|
||||
device = video_embeddings.device
|
||||
tokenizer = self._tokenizer
|
||||
tokenizer = cached_tokenizer_from_config(self.model_config)
|
||||
|
||||
# Generate video replacement token IDs using get_video_repl
|
||||
# This tokenizes each frame separator independently, then uses pre-tokenized
|
||||
|
||||
@@ -347,7 +347,7 @@ class NemotronVLProcessingInfo(BaseInternVLProcessingInfo):
|
||||
|
||||
def get_image_processor(self, **kwargs: object):
|
||||
return cached_image_processor_from_config(
|
||||
self.ctx.renderer_config,
|
||||
self.ctx.model_config,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
@@ -193,7 +193,7 @@ class PixtralProcessorAdapter:
|
||||
|
||||
class PixtralProcessingInfo(BaseProcessingInfo):
|
||||
def get_tokenizer(self) -> MistralTokenizer:
|
||||
tokenizer = cached_tokenizer_from_config(self.ctx.renderer_config)
|
||||
tokenizer = cached_tokenizer_from_config(self.ctx.model_config)
|
||||
if not isinstance(tokenizer, MistralTokenizer):
|
||||
raise ValueError("This model requires `--tokenizer-mode mistral`")
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ from mistral_common.tokens.tokenizers.audio import Audio, AudioEncoder
|
||||
from transformers import BatchFeature, TensorType, WhisperConfig
|
||||
from transformers.tokenization_utils_base import TextInput
|
||||
|
||||
from vllm.config import RendererConfig, SpeechToTextConfig, VllmConfig
|
||||
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
|
||||
from vllm.config.multimodal import BaseDummyOptions
|
||||
from vllm.inputs.data import PromptType
|
||||
from vllm.logger import init_logger
|
||||
@@ -176,7 +176,7 @@ class VoxtralProcessorAdapter:
|
||||
|
||||
class VoxtralProcessingInfo(BaseProcessingInfo):
|
||||
def get_tokenizer(self) -> MistralTokenizer:
|
||||
tokenizer = cached_tokenizer_from_config(self.ctx.renderer_config)
|
||||
tokenizer = cached_tokenizer_from_config(self.ctx.model_config)
|
||||
if not isinstance(tokenizer, MistralTokenizer):
|
||||
raise ValueError("This model requires `--tokenizer-mode mistral`")
|
||||
|
||||
@@ -339,7 +339,7 @@ class VoxtralForConditionalGeneration(
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.tokenizer = cached_tokenizer_from_config(vllm_config.renderer_config)
|
||||
self.tokenizer = cached_tokenizer_from_config(vllm_config.model_config)
|
||||
|
||||
# update quant config to so that ignored module and target module names
|
||||
# match the vLLM model names
|
||||
@@ -450,11 +450,9 @@ class VoxtralForConditionalGeneration(
|
||||
|
||||
@classmethod
|
||||
def get_speech_to_text_config(
|
||||
cls,
|
||||
renderer_config: RendererConfig,
|
||||
task_type: str,
|
||||
cls, model_config: ModelConfig, task_type: str
|
||||
) -> SpeechToTextConfig:
|
||||
tokenizer = cached_tokenizer_from_config(renderer_config)
|
||||
tokenizer = cached_tokenizer_from_config(model_config)
|
||||
audio_config = tokenizer.instruct.audio_encoder.audio_config
|
||||
max_audio_clip_s = audio_config.chunk_length_s
|
||||
sample_rate = audio_config.sampling_rate
|
||||
@@ -470,17 +468,17 @@ class VoxtralForConditionalGeneration(
|
||||
def get_generation_prompt(
|
||||
cls,
|
||||
audio: np.ndarray,
|
||||
renderer_config: RendererConfig, # not needed here
|
||||
model_config: ModelConfig,
|
||||
stt_config: SpeechToTextConfig,
|
||||
language: str | None,
|
||||
task_type: Literal["transcribe", "translate"],
|
||||
request_prompt: str,
|
||||
to_language: str | None,
|
||||
) -> PromptType:
|
||||
tokenizer = cached_tokenizer_from_config(renderer_config)
|
||||
tokenizer = cached_tokenizer_from_config(model_config)
|
||||
audio = Audio(audio, int(stt_config.sample_rate), format="wav") # lossless
|
||||
req = TranscriptionRequest(
|
||||
model=renderer_config.model_config.model,
|
||||
model=model_config.model,
|
||||
audio=RawAudio.from_audio(audio),
|
||||
language=language,
|
||||
)
|
||||
@@ -496,14 +494,14 @@ class VoxtralForConditionalGeneration(
|
||||
cls,
|
||||
audio_duration_s: float,
|
||||
stt_config: SpeechToTextConfig,
|
||||
renderer_config: RendererConfig,
|
||||
model_config: ModelConfig,
|
||||
) -> int | None:
|
||||
"""
|
||||
Map from audio duration to number of audio tokens produced by the ASR
|
||||
model, without running a forward pass.
|
||||
This is used for estimating the amount of processing for this audio.
|
||||
"""
|
||||
tokenizer = cached_tokenizer_from_config(renderer_config)
|
||||
tokenizer = cached_tokenizer_from_config(model_config)
|
||||
adapter = VoxtralProcessorAdapter(tokenizer)
|
||||
return adapter.get_num_audio_tokens(
|
||||
int(audio_duration_s * stt_config.sample_rate)
|
||||
|
||||
@@ -19,7 +19,7 @@ from transformers.models.whisper.modeling_whisper import sinusoids
|
||||
from vllm.attention.backends.abstract import AttentionType
|
||||
from vllm.attention.layer import Attention, MultiHeadAttention
|
||||
from vllm.attention.layers.cross_attention import CrossAttention
|
||||
from vllm.config import CacheConfig, RendererConfig, SpeechToTextConfig, VllmConfig
|
||||
from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
|
||||
from vllm.config.multimodal import BaseDummyOptions
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.inputs.data import PromptType
|
||||
@@ -811,7 +811,7 @@ class WhisperForConditionalGeneration(
|
||||
def get_generation_prompt(
|
||||
cls,
|
||||
audio: np.ndarray,
|
||||
renderer_config: RendererConfig, # not needed here
|
||||
model_config: ModelConfig, # not needed here
|
||||
stt_config: SpeechToTextConfig,
|
||||
language: str | None,
|
||||
task_type: Literal["transcribe", "translate"],
|
||||
@@ -847,11 +847,9 @@ class WhisperForConditionalGeneration(
|
||||
|
||||
@classmethod
|
||||
def get_speech_to_text_config(
|
||||
cls,
|
||||
renderer_config: RendererConfig,
|
||||
task_type: str,
|
||||
cls, model_config: ModelConfig, task_type: str
|
||||
) -> SpeechToTextConfig:
|
||||
processor = cached_processor_from_config(renderer_config)
|
||||
processor = cached_processor_from_config(model_config)
|
||||
|
||||
return SpeechToTextConfig(
|
||||
max_audio_clip_s=processor.feature_extractor.chunk_length,
|
||||
@@ -863,9 +861,9 @@ class WhisperForConditionalGeneration(
|
||||
cls,
|
||||
audio_duration_s: float,
|
||||
stt_config: SpeechToTextConfig,
|
||||
renderer_config: RendererConfig,
|
||||
model_config: ModelConfig,
|
||||
) -> int | None:
|
||||
processor = cached_processor_from_config(renderer_config)
|
||||
processor = cached_processor_from_config(model_config)
|
||||
hop_length = processor.feature_extractor.hop_length
|
||||
assert hop_length is not None
|
||||
# NOTE(NickLucche) user can't pass encoder
|
||||
|
||||
+12
-10
@@ -31,7 +31,7 @@ from .inputs import (
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import ModelConfig, RendererConfig, VllmConfig
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
|
||||
from .processing import ResolvedPromptUpdate
|
||||
from .registry import MultiModalRegistry
|
||||
@@ -561,13 +561,13 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):
|
||||
|
||||
|
||||
def _enable_processor_cache(
|
||||
renderer_config: "RendererConfig",
|
||||
model_config: "ModelConfig",
|
||||
mm_registry: "MultiModalRegistry",
|
||||
) -> bool:
|
||||
if not mm_registry.supports_multimodal_inputs(renderer_config):
|
||||
if not mm_registry.supports_multimodal_inputs(model_config):
|
||||
return False
|
||||
|
||||
mm_config = renderer_config.model_config.get_multimodal_config()
|
||||
mm_config = model_config.get_multimodal_config()
|
||||
return mm_config.mm_processor_cache_gb > 0
|
||||
|
||||
|
||||
@@ -599,7 +599,7 @@ def processor_cache_from_config(
|
||||
"""Return a `BaseMultiModalProcessorCache`, if enabled."""
|
||||
model_config = vllm_config.model_config
|
||||
|
||||
if not _enable_processor_cache(vllm_config.renderer_config, mm_registry):
|
||||
if not _enable_processor_cache(model_config, mm_registry):
|
||||
return None
|
||||
|
||||
if not _enable_ipc_cache(vllm_config):
|
||||
@@ -611,14 +611,14 @@ def processor_cache_from_config(
|
||||
|
||||
|
||||
def processor_only_cache_from_config(
|
||||
renderer_config: "RendererConfig",
|
||||
model_config: "ModelConfig",
|
||||
mm_registry: "MultiModalRegistry",
|
||||
):
|
||||
"""Return a `MultiModalProcessorOnlyCache`, if enabled."""
|
||||
if not _enable_processor_cache(renderer_config, mm_registry):
|
||||
if not _enable_processor_cache(model_config, mm_registry):
|
||||
return None
|
||||
|
||||
return MultiModalProcessorOnlyCache(renderer_config.model_config)
|
||||
return MultiModalProcessorOnlyCache(model_config)
|
||||
|
||||
|
||||
class BaseMultiModalReceiverCache(
|
||||
@@ -787,7 +787,7 @@ def engine_receiver_cache_from_config(
|
||||
"""
|
||||
model_config = vllm_config.model_config
|
||||
|
||||
if not _enable_processor_cache(vllm_config.renderer_config, mm_registry):
|
||||
if not _enable_processor_cache(model_config, mm_registry):
|
||||
return None
|
||||
|
||||
if not _enable_ipc_cache(vllm_config):
|
||||
@@ -809,7 +809,9 @@ def worker_receiver_cache_from_config(
|
||||
Return a `BaseMultiModalReceiverCache` only when IPC caching is enabled and
|
||||
mm_processor_cache_type=="shm".
|
||||
"""
|
||||
if not _enable_processor_cache(vllm_config.renderer_config, mm_registry):
|
||||
model_config = vllm_config.model_config
|
||||
|
||||
if not _enable_processor_cache(model_config, mm_registry):
|
||||
return None
|
||||
|
||||
if not _enable_ipc_cache(vllm_config):
|
||||
|
||||
@@ -23,7 +23,7 @@ import torch
|
||||
from typing_extensions import TypeVar, assert_never
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.transformers_utils.processor import cached_processor_from_config
|
||||
from vllm.utils.collection_utils import flatten_2d_lists, full_groupby
|
||||
from vllm.utils.func_utils import get_allowed_kwarg_only_overrides
|
||||
@@ -53,7 +53,7 @@ if TYPE_CHECKING:
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
from transformers.processing_utils import ProcessorMixin
|
||||
|
||||
from vllm.config import ModelConfig, RendererConfig
|
||||
from vllm.config import ModelConfig
|
||||
|
||||
from .cache import BaseMultiModalProcessorCache
|
||||
from .profiling import BaseDummyInputsBuilder
|
||||
@@ -63,7 +63,6 @@ else:
|
||||
ProcessorMixin = object
|
||||
|
||||
ModelConfig = object
|
||||
RendererConfig = object
|
||||
|
||||
BaseMultiModalProcessorCache = object
|
||||
|
||||
@@ -946,29 +945,12 @@ class InputProcessingContext:
|
||||
modify the inputs.
|
||||
"""
|
||||
|
||||
renderer_config: RendererConfig
|
||||
"""The configuration of the renderer."""
|
||||
model_config: ModelConfig
|
||||
"""The configuration of the model."""
|
||||
|
||||
tokenizer: TokenizerLike | None
|
||||
"""The tokenizer used to tokenize the inputs."""
|
||||
|
||||
@classmethod
|
||||
def from_config(
|
||||
cls,
|
||||
renderer_config: RendererConfig,
|
||||
*,
|
||||
tokenizer: TokenizerLike | None = None,
|
||||
):
|
||||
if tokenizer is None and not renderer_config.skip_tokenizer_init:
|
||||
tokenizer = cached_tokenizer_from_config(renderer_config)
|
||||
|
||||
return cls(renderer_config, tokenizer)
|
||||
|
||||
@property
|
||||
def model_config(self) -> ModelConfig:
|
||||
"""The configuration of the model."""
|
||||
return self.renderer_config.model_config
|
||||
|
||||
def get_tokenizer(self) -> TokenizerLike:
|
||||
if self.tokenizer is None:
|
||||
raise ValueError(
|
||||
@@ -1065,7 +1047,7 @@ class InputProcessingContext:
|
||||
typ = ProcessorMixin
|
||||
|
||||
return cached_processor_from_config(
|
||||
self.renderer_config,
|
||||
self.model_config,
|
||||
processor_cls=typ,
|
||||
tokenizer=self.tokenizer,
|
||||
**kwargs,
|
||||
|
||||
+34
-30
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Generic, Protocol, TypeVar, cast
|
||||
|
||||
from vllm.config.multimodal import BaseDummyOptions
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
|
||||
|
||||
from .cache import BaseMultiModalProcessorCache
|
||||
from .processing import (
|
||||
@@ -22,7 +22,7 @@ from .profiling import (
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import ModelConfig, RendererConfig
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.model_executor.models.interfaces import SupportsMultiModal
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -114,18 +114,17 @@ class MultiModalRegistry:
|
||||
|
||||
return mm_options if len(mm_options) > 0 else None
|
||||
|
||||
def supports_multimodal_inputs(self, renderer_config: "RendererConfig") -> bool:
|
||||
def supports_multimodal_inputs(self, model_config: "ModelConfig") -> bool:
|
||||
"""
|
||||
Checks if the model supports multimodal inputs.
|
||||
Returns True if the model is multimodal with any non-zero supported
|
||||
modalities, otherwise returns False, effectively running in
|
||||
text-only mode.
|
||||
"""
|
||||
model_config = renderer_config.model_config
|
||||
if not model_config.is_multimodal_model:
|
||||
return False
|
||||
|
||||
info = self._create_processing_info(renderer_config, tokenizer=None)
|
||||
info = self._create_processing_info(model_config, tokenizer=None)
|
||||
supported_modalities = info.get_supported_mm_limits()
|
||||
|
||||
mm_config = model_config.get_multimodal_config()
|
||||
@@ -145,7 +144,7 @@ class MultiModalRegistry:
|
||||
|
||||
def get_max_tokens_per_item_by_modality(
|
||||
self,
|
||||
renderer_config: "RendererConfig",
|
||||
model_config: "ModelConfig",
|
||||
*,
|
||||
cache: BaseMultiModalProcessorCache | None = None,
|
||||
profiler_limits: Mapping[str, int] | None = None,
|
||||
@@ -154,11 +153,10 @@ class MultiModalRegistry:
|
||||
Get the maximum number of tokens per data item from each modality based
|
||||
on underlying model configuration.
|
||||
"""
|
||||
model_config = renderer_config.model_config
|
||||
if not model_config.is_multimodal_model:
|
||||
return {}
|
||||
|
||||
processor = self.create_processor(renderer_config, cache=cache)
|
||||
processor = self.create_processor(model_config, cache=cache)
|
||||
profiler: MultiModalProfiler = MultiModalProfiler(processor)
|
||||
|
||||
seq_len = model_config.max_model_len
|
||||
@@ -173,7 +171,7 @@ class MultiModalRegistry:
|
||||
|
||||
def get_mm_limits_per_prompt(
|
||||
self,
|
||||
renderer_config: "RendererConfig",
|
||||
model_config: "ModelConfig",
|
||||
*,
|
||||
cache: BaseMultiModalProcessorCache | None = None,
|
||||
) -> Mapping[str, int]:
|
||||
@@ -181,11 +179,10 @@ class MultiModalRegistry:
|
||||
Get the maximum number of multi-modal input instances for each modality
|
||||
that are allowed per prompt for a model class.
|
||||
"""
|
||||
model_config = renderer_config.model_config
|
||||
if not model_config.is_multimodal_model:
|
||||
return {}
|
||||
|
||||
processor = self.create_processor(renderer_config, cache=cache)
|
||||
processor = self.create_processor(model_config, cache=cache)
|
||||
profiler: MultiModalProfiler = MultiModalProfiler(processor)
|
||||
return profiler.get_mm_limits()
|
||||
|
||||
@@ -231,21 +228,30 @@ class MultiModalRegistry:
|
||||
assert hasattr(model_cls, "_processor_factory")
|
||||
return cast("SupportsMultiModal", model_cls)
|
||||
|
||||
def _create_processing_ctx(
|
||||
self,
|
||||
model_config: "ModelConfig",
|
||||
tokenizer: TokenizerLike | None = None,
|
||||
) -> InputProcessingContext:
|
||||
if tokenizer is None and not model_config.skip_tokenizer_init:
|
||||
tokenizer = cached_tokenizer_from_config(model_config)
|
||||
|
||||
return InputProcessingContext(model_config, tokenizer)
|
||||
|
||||
def _create_processing_info(
|
||||
self,
|
||||
renderer_config: "RendererConfig",
|
||||
model_config: "ModelConfig",
|
||||
*,
|
||||
tokenizer: TokenizerLike | None = None,
|
||||
) -> BaseProcessingInfo:
|
||||
model_cls = self._get_model_cls(renderer_config.model_config)
|
||||
model_cls = self._get_model_cls(model_config)
|
||||
factories = model_cls._processor_factory
|
||||
|
||||
ctx = InputProcessingContext.from_config(renderer_config, tokenizer=tokenizer)
|
||||
ctx = self._create_processing_ctx(model_config, tokenizer)
|
||||
return factories.info(ctx)
|
||||
|
||||
def create_processor(
|
||||
self,
|
||||
renderer_config: "RendererConfig",
|
||||
model_config: "ModelConfig",
|
||||
*,
|
||||
tokenizer: TokenizerLike | None = None,
|
||||
cache: BaseMultiModalProcessorCache | None = None,
|
||||
@@ -253,19 +259,19 @@ class MultiModalRegistry:
|
||||
"""
|
||||
Create a multi-modal processor for a specific model and tokenizer.
|
||||
"""
|
||||
model_config = renderer_config.model_config
|
||||
if not model_config.is_multimodal_model:
|
||||
raise ValueError(f"{model_config.model} is not a multimodal model")
|
||||
|
||||
model_cls = self._get_model_cls(model_config)
|
||||
factories = model_cls._processor_factory
|
||||
|
||||
ctx = InputProcessingContext.from_config(renderer_config, tokenizer=tokenizer)
|
||||
ctx = self._create_processing_ctx(model_config, tokenizer)
|
||||
|
||||
return factories.build_processor(ctx, cache=cache)
|
||||
|
||||
def get_decoder_dummy_data(
|
||||
self,
|
||||
renderer_config: "RendererConfig",
|
||||
model_config: "ModelConfig",
|
||||
seq_len: int,
|
||||
mm_counts: Mapping[str, int] | None = None,
|
||||
*,
|
||||
@@ -274,15 +280,15 @@ class MultiModalRegistry:
|
||||
"""
|
||||
Create dummy data for profiling the memory usage of a model.
|
||||
|
||||
The model is identified by `renderer_config`.
|
||||
The model is identified by `model_config`.
|
||||
"""
|
||||
processor = self.create_processor(renderer_config, cache=cache)
|
||||
processor = self.create_processor(model_config, cache=cache)
|
||||
profiler: MultiModalProfiler = MultiModalProfiler(processor)
|
||||
|
||||
# Extract configurable options from multimodal config.
|
||||
# Only include modalities that use advanced option types so legacy
|
||||
# count-only behavior remains unchanged.
|
||||
mm_options = self._extract_mm_options(renderer_config.model_config)
|
||||
mm_options = self._extract_mm_options(model_config)
|
||||
|
||||
dummy_data = profiler.get_decoder_dummy_data(seq_len, mm_counts, mm_options)
|
||||
|
||||
@@ -298,7 +304,7 @@ class MultiModalRegistry:
|
||||
|
||||
def get_encoder_dummy_data(
|
||||
self,
|
||||
renderer_config: "RendererConfig",
|
||||
model_config: "ModelConfig",
|
||||
seq_len: int,
|
||||
mm_counts: Mapping[str, int] | None = None,
|
||||
*,
|
||||
@@ -307,15 +313,15 @@ class MultiModalRegistry:
|
||||
"""
|
||||
Create dummy data for profiling the memory usage of a model.
|
||||
|
||||
The model is identified by `renderer_config`.
|
||||
The model is identified by `model_config`.
|
||||
"""
|
||||
processor = self.create_processor(renderer_config, cache=cache)
|
||||
processor = self.create_processor(model_config, cache=cache)
|
||||
profiler: MultiModalProfiler = MultiModalProfiler(processor)
|
||||
|
||||
# Extract configurable options from multimodal config.
|
||||
# Only include modalities that use advanced option types so legacy
|
||||
# count-only behavior remains unchanged.
|
||||
mm_options = self._extract_mm_options(renderer_config.model_config)
|
||||
mm_options = self._extract_mm_options(model_config)
|
||||
|
||||
dummy_data = profiler.get_encoder_dummy_data(seq_len, mm_counts, mm_options)
|
||||
|
||||
@@ -330,15 +336,13 @@ class MultiModalRegistry:
|
||||
|
||||
return dummy_data
|
||||
|
||||
def get_encdec_max_encoder_len(self, renderer_config: "RendererConfig") -> int:
|
||||
def get_encdec_max_encoder_len(self, model_config: "ModelConfig") -> int:
|
||||
"""
|
||||
Get the maximum length of the encoder input for encoder-decoder models.
|
||||
"""
|
||||
model_config = renderer_config.model_config
|
||||
if not model_config.is_encoder_decoder:
|
||||
return 0
|
||||
|
||||
max_tokens = self.get_max_tokens_per_item_by_modality(renderer_config)
|
||||
max_tokens = self.get_max_tokens_per_item_by_modality(model_config)
|
||||
if not max_tokens:
|
||||
# TODO - this function assumes encoder-decoder models are
|
||||
# multimodal. This will need to change when adding support for more
|
||||
|
||||
+12
-12
@@ -24,7 +24,7 @@ from vllm.utils.import_utils import resolve_obj_by_qualname
|
||||
from .protocol import TokenizerLike
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import RendererConfig
|
||||
from vllm.config import ModelConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -205,18 +205,18 @@ def get_tokenizer(
|
||||
cached_get_tokenizer = lru_cache(get_tokenizer)
|
||||
|
||||
|
||||
def cached_tokenizer_from_config(renderer_config: "RendererConfig", **kwargs):
|
||||
def cached_tokenizer_from_config(model_config: "ModelConfig", **kwargs):
|
||||
return cached_get_tokenizer(
|
||||
renderer_config.tokenizer,
|
||||
tokenizer_mode=renderer_config.tokenizer_mode,
|
||||
revision=renderer_config.tokenizer_revision,
|
||||
trust_remote_code=renderer_config.trust_remote_code,
|
||||
model_config.tokenizer,
|
||||
tokenizer_mode=model_config.tokenizer_mode,
|
||||
revision=model_config.tokenizer_revision,
|
||||
trust_remote_code=model_config.trust_remote_code,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def init_tokenizer_from_config(renderer_config: "RendererConfig"):
|
||||
runner_type = renderer_config.model_config.runner_type
|
||||
def init_tokenizer_from_config(model_config: "ModelConfig"):
|
||||
runner_type = model_config.runner_type
|
||||
if runner_type == "generate" or runner_type == "draft":
|
||||
truncation_side = "left"
|
||||
elif runner_type == "pooling":
|
||||
@@ -225,9 +225,9 @@ def init_tokenizer_from_config(renderer_config: "RendererConfig"):
|
||||
assert_never(runner_type)
|
||||
|
||||
return get_tokenizer(
|
||||
renderer_config.tokenizer,
|
||||
tokenizer_mode=renderer_config.tokenizer_mode,
|
||||
trust_remote_code=renderer_config.trust_remote_code,
|
||||
revision=renderer_config.tokenizer_revision,
|
||||
model_config.tokenizer,
|
||||
tokenizer_mode=model_config.tokenizer_mode,
|
||||
trust_remote_code=model_config.trust_remote_code,
|
||||
revision=model_config.tokenizer_revision,
|
||||
truncation_side=truncation_side,
|
||||
)
|
||||
|
||||
@@ -23,7 +23,7 @@ from vllm.transformers_utils.utils import convert_model_repo_to_path
|
||||
from vllm.utils.func_utils import get_allowed_kwarg_only_overrides
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import ModelConfig, RendererConfig
|
||||
from vllm.config import ModelConfig
|
||||
|
||||
_P = TypeVar("_P", bound=ProcessorMixin, default=ProcessorMixin)
|
||||
_V = TypeVar("_V", bound=BaseVideoProcessor, default=BaseVideoProcessor)
|
||||
@@ -233,18 +233,17 @@ def cached_get_processor_without_dynamic_kwargs(
|
||||
|
||||
|
||||
def cached_processor_from_config(
|
||||
renderer_config: "RendererConfig",
|
||||
model_config: "ModelConfig",
|
||||
processor_cls: type[_P] | tuple[type[_P], ...] = ProcessorMixin,
|
||||
**kwargs: Any,
|
||||
) -> _P:
|
||||
model_config = renderer_config.model_config
|
||||
if is_gguf(model_config.model):
|
||||
assert not is_gguf(renderer_config.tokenizer), (
|
||||
assert not is_gguf(model_config.tokenizer), (
|
||||
"For multimodal GGUF models, the original tokenizer "
|
||||
"should be used to correctly load processor."
|
||||
)
|
||||
model = renderer_config.tokenizer
|
||||
revision = renderer_config.tokenizer_revision
|
||||
model = model_config.tokenizer
|
||||
revision = model_config.tokenizer_revision
|
||||
else:
|
||||
model = model_config.model
|
||||
revision = model_config.revision
|
||||
@@ -298,11 +297,9 @@ cached_get_feature_extractor = lru_cache(get_feature_extractor)
|
||||
|
||||
|
||||
def cached_feature_extractor_from_config(
|
||||
renderer_config: "RendererConfig",
|
||||
model_config: "ModelConfig",
|
||||
**kwargs: Any,
|
||||
):
|
||||
model_config = renderer_config.model_config
|
||||
|
||||
return cached_get_feature_extractor(
|
||||
model_config.model,
|
||||
revision=model_config.revision,
|
||||
@@ -351,17 +348,16 @@ cached_get_image_processor = lru_cache(get_image_processor)
|
||||
|
||||
|
||||
def cached_image_processor_from_config(
|
||||
renderer_config: "RendererConfig",
|
||||
model_config: "ModelConfig",
|
||||
**kwargs: Any,
|
||||
):
|
||||
model_config = renderer_config.model_config
|
||||
if is_gguf(model_config.model):
|
||||
assert not is_gguf(renderer_config.tokenizer), (
|
||||
assert not is_gguf(model_config.tokenizer), (
|
||||
"For multimodal GGUF models, the original tokenizer "
|
||||
"should be used to correctly load image processor."
|
||||
)
|
||||
model = renderer_config.tokenizer
|
||||
revision = renderer_config.tokenizer_revision
|
||||
model = model_config.tokenizer
|
||||
revision = model_config.tokenizer_revision
|
||||
else:
|
||||
model = model_config.model
|
||||
revision = model_config.revision
|
||||
@@ -415,12 +411,10 @@ cached_get_video_processor = lru_cache(get_video_processor)
|
||||
|
||||
|
||||
def cached_video_processor_from_config(
|
||||
renderer_config: "RendererConfig",
|
||||
model_config: "ModelConfig",
|
||||
processor_cls: type[_V] | None = None,
|
||||
**kwargs: Any,
|
||||
):
|
||||
model_config = renderer_config.model_config
|
||||
|
||||
return cached_get_video_processor(
|
||||
model_config.model,
|
||||
revision=model_config.revision,
|
||||
|
||||
@@ -10,7 +10,7 @@ from vllm.multimodal import MultiModalRegistry
|
||||
from vllm.v1.request import Request
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import RendererConfig, SchedulerConfig
|
||||
from vllm.config import ModelConfig, SchedulerConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -250,7 +250,7 @@ class EncoderCacheManager:
|
||||
|
||||
|
||||
def compute_encoder_budget(
|
||||
renderer_config: "RendererConfig",
|
||||
model_config: "ModelConfig",
|
||||
scheduler_config: "SchedulerConfig",
|
||||
mm_registry: MultiModalRegistry,
|
||||
) -> tuple[int, int]:
|
||||
@@ -263,9 +263,9 @@ def compute_encoder_budget(
|
||||
- Space budget for encoder cache size, measured in number of tokens
|
||||
from the input sequence.
|
||||
"""
|
||||
if mm_registry.supports_multimodal_inputs(renderer_config):
|
||||
if mm_registry.supports_multimodal_inputs(model_config):
|
||||
max_tokens_by_modality = mm_registry.get_max_tokens_per_item_by_modality(
|
||||
renderer_config
|
||||
model_config
|
||||
)
|
||||
|
||||
return compute_mm_encoder_budget(
|
||||
|
||||
@@ -164,7 +164,7 @@ class Scheduler(SchedulerInterface):
|
||||
# This can be changed when we make encoder cache for embedding caching
|
||||
# across requests.
|
||||
encoder_compute_budget, encoder_cache_size = compute_encoder_budget(
|
||||
renderer_config=vllm_config.renderer_config,
|
||||
model_config=vllm_config.model_config,
|
||||
scheduler_config=vllm_config.scheduler_config,
|
||||
mm_registry=mm_registry,
|
||||
)
|
||||
|
||||
@@ -91,7 +91,6 @@ class AsyncLLM(EngineClient):
|
||||
# Ensure we can serialize custom transformer configs
|
||||
maybe_register_config_serialize_by_value()
|
||||
|
||||
self.renderer_config = vllm_config.renderer_config
|
||||
self.model_config = vllm_config.model_config
|
||||
self.vllm_config = vllm_config
|
||||
self.observability_config = vllm_config.observability_config
|
||||
@@ -109,15 +108,15 @@ class AsyncLLM(EngineClient):
|
||||
"enabling logging without default stat loggers."
|
||||
)
|
||||
|
||||
if self.renderer_config.skip_tokenizer_init:
|
||||
if self.model_config.skip_tokenizer_init:
|
||||
tokenizer = None
|
||||
else:
|
||||
tokenizer = init_tokenizer_from_config(self.renderer_config)
|
||||
tokenizer = init_tokenizer_from_config(self.model_config)
|
||||
|
||||
self.input_processor = InputProcessor(self.vllm_config, tokenizer)
|
||||
self.io_processor = get_io_processor(
|
||||
self.vllm_config,
|
||||
self.renderer_config.io_processor_plugin,
|
||||
self.model_config.io_processor_plugin,
|
||||
)
|
||||
|
||||
# OutputProcessor (converts EngineCoreOutputs --> RequestOutput).
|
||||
|
||||
@@ -43,7 +43,6 @@ class InputProcessor:
|
||||
mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
|
||||
) -> None:
|
||||
self.vllm_config = vllm_config
|
||||
self.renderer_config = vllm_config.renderer_config
|
||||
self.model_config = vllm_config.model_config
|
||||
self.cache_config = vllm_config.cache_config
|
||||
self.lora_config = vllm_config.lora_config
|
||||
@@ -55,7 +54,7 @@ class InputProcessor:
|
||||
self.mm_processor_cache = processor_cache_from_config(vllm_config, mm_registry)
|
||||
|
||||
self.input_preprocessor = InputPreprocessor(
|
||||
self.renderer_config,
|
||||
self.model_config,
|
||||
tokenizer,
|
||||
mm_registry,
|
||||
mm_processor_cache=self.mm_processor_cache,
|
||||
@@ -253,7 +252,7 @@ class InputProcessor:
|
||||
if not params.structured_outputs or not self.structured_outputs_config:
|
||||
return
|
||||
|
||||
if self.renderer_config.skip_tokenizer_init and params.structured_outputs:
|
||||
if self.model_config.skip_tokenizer_init and params.structured_outputs:
|
||||
raise ValueError(
|
||||
"Structured outputs requires a tokenizer so it can't be used with 'skip_tokenizer_init'" # noqa: E501
|
||||
)
|
||||
@@ -583,7 +582,7 @@ class InputProcessor:
|
||||
if prompt_type == "encoder" and model_config.is_multimodal_model:
|
||||
mm_registry = self.input_preprocessor.mm_registry
|
||||
mm_processor = mm_registry.create_processor(
|
||||
self.renderer_config,
|
||||
model_config,
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
assert isinstance(mm_processor, EncDecMultiModalProcessor)
|
||||
|
||||
@@ -60,7 +60,6 @@ class LLMEngine:
|
||||
) -> None:
|
||||
self.vllm_config = vllm_config
|
||||
self.observability_config = vllm_config.observability_config
|
||||
self.renderer_config = vllm_config.renderer_config
|
||||
self.model_config = vllm_config.model_config
|
||||
self.cache_config = vllm_config.cache_config
|
||||
|
||||
@@ -84,15 +83,15 @@ class LLMEngine:
|
||||
self.dp_group = None
|
||||
self.should_execute_dummy_batch = False
|
||||
|
||||
if self.renderer_config.skip_tokenizer_init:
|
||||
if self.model_config.skip_tokenizer_init:
|
||||
tokenizer = None
|
||||
else:
|
||||
tokenizer = init_tokenizer_from_config(self.renderer_config)
|
||||
tokenizer = init_tokenizer_from_config(self.model_config)
|
||||
|
||||
self.input_processor = InputProcessor(self.vllm_config, tokenizer)
|
||||
self.io_processor = get_io_processor(
|
||||
self.vllm_config,
|
||||
self.renderer_config.io_processor_plugin,
|
||||
self.model_config.io_processor_plugin,
|
||||
)
|
||||
|
||||
# OutputProcessor (convert EngineCoreOutputs --> RequestOutput).
|
||||
|
||||
@@ -85,7 +85,7 @@ class EagleProposer:
|
||||
# Multi-modal data support
|
||||
self.mm_registry = MULTIMODAL_REGISTRY
|
||||
self.supports_mm_inputs = self.mm_registry.supports_multimodal_inputs(
|
||||
vllm_config.renderer_config
|
||||
vllm_config.model_config
|
||||
)
|
||||
|
||||
self.attn_metadata_builder: AttentionMetadataBuilder | None = None
|
||||
|
||||
@@ -63,7 +63,7 @@ class StructuredOutputManager:
|
||||
max_workers = max(1, min(multiprocessing.cpu_count() // 2, 8))
|
||||
self.executor_for_fillmask = ThreadPoolExecutor(max_workers=max_workers)
|
||||
|
||||
if not vllm_config.renderer_config.skip_tokenizer_init:
|
||||
if not self.vllm_config.model_config.skip_tokenizer_init:
|
||||
# The default max_workers if not specified is the number of
|
||||
# CPUs * 5, which is way too high since these tasks are CPU-bound,
|
||||
# not I/O bound. We also know we would never dominate CPU usage
|
||||
@@ -71,15 +71,21 @@ class StructuredOutputManager:
|
||||
# of CPUs.
|
||||
max_workers = max(1, (multiprocessing.cpu_count() + 1) // 2)
|
||||
self.executor = ThreadPoolExecutor(max_workers=max_workers)
|
||||
self.tokenizer = init_tokenizer_from_config(vllm_config.renderer_config)
|
||||
reasoning_parser = vllm_config.structured_outputs_config.reasoning_parser
|
||||
self.tokenizer = init_tokenizer_from_config(
|
||||
model_config=self.vllm_config.model_config
|
||||
)
|
||||
reasoning_parser = (
|
||||
self.vllm_config.structured_outputs_config.reasoning_parser
|
||||
)
|
||||
reasoning_parser_plugin = (
|
||||
vllm_config.structured_outputs_config.reasoning_parser_plugin
|
||||
self.vllm_config.structured_outputs_config.reasoning_parser_plugin
|
||||
)
|
||||
if reasoning_parser_plugin and len(reasoning_parser_plugin) > 3:
|
||||
ReasoningParserManager.import_reasoning_parser(reasoning_parser_plugin)
|
||||
|
||||
reasoning_parser = vllm_config.structured_outputs_config.reasoning_parser
|
||||
reasoning_parser = (
|
||||
self.vllm_config.structured_outputs_config.reasoning_parser
|
||||
)
|
||||
if reasoning_parser:
|
||||
reasoner_cls = ReasoningParserManager.get_reasoning_parser(
|
||||
reasoning_parser
|
||||
@@ -87,7 +93,7 @@ class StructuredOutputManager:
|
||||
self.reasoner = reasoner_cls(tokenizer=self.tokenizer)
|
||||
|
||||
self.enable_in_reasoning = (
|
||||
vllm_config.structured_outputs_config.enable_in_reasoning
|
||||
self.vllm_config.structured_outputs_config.enable_in_reasoning
|
||||
)
|
||||
|
||||
def grammar_init(self, request: Request) -> None:
|
||||
|
||||
@@ -271,7 +271,6 @@ class GPUModelRunner(
|
||||
device: torch.device,
|
||||
):
|
||||
self.vllm_config = vllm_config
|
||||
self.renderer_config = vllm_config.renderer_config
|
||||
self.model_config = vllm_config.model_config
|
||||
self.cache_config = vllm_config.cache_config
|
||||
self.compilation_config = vllm_config.compilation_config
|
||||
@@ -336,7 +335,7 @@ class GPUModelRunner(
|
||||
self.uses_mrope = model_config.uses_mrope
|
||||
self.uses_xdrope_dim = model_config.uses_xdrope_dim
|
||||
self.supports_mm_inputs = self.mm_registry.supports_multimodal_inputs(
|
||||
self.renderer_config
|
||||
model_config
|
||||
)
|
||||
|
||||
if self.model_config.is_encoder_decoder:
|
||||
@@ -559,7 +558,7 @@ class GPUModelRunner(
|
||||
|
||||
self.mm_budget = (
|
||||
MultiModalBudget(
|
||||
self.renderer_config,
|
||||
self.model_config,
|
||||
self.scheduler_config,
|
||||
self.mm_registry,
|
||||
)
|
||||
@@ -3874,7 +3873,7 @@ class GPUModelRunner(
|
||||
assert self.mm_budget is not None
|
||||
|
||||
dummy_decoder_data = self.mm_registry.get_decoder_dummy_data(
|
||||
renderer_config=self.renderer_config,
|
||||
model_config=self.model_config,
|
||||
seq_len=self.max_model_len,
|
||||
mm_counts={modality: 1},
|
||||
cache=self.mm_budget.cache,
|
||||
|
||||
@@ -143,7 +143,6 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
original_parallel_config: ParallelConfig | None = None,
|
||||
):
|
||||
self.vllm_config = vllm_config
|
||||
self.renderer_config = vllm_config.renderer_config
|
||||
self.model_config = vllm_config.model_config
|
||||
self.cache_config = vllm_config.cache_config
|
||||
self.lora_config = vllm_config.lora_config
|
||||
@@ -223,7 +222,7 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
self.mm_registry = MULTIMODAL_REGISTRY
|
||||
self.uses_mrope = model_config.uses_mrope
|
||||
self.supports_mm_inputs = self.mm_registry.supports_multimodal_inputs(
|
||||
self.renderer_config
|
||||
model_config
|
||||
)
|
||||
# TODO: Support M-RoPE (e.g, Qwen2-VL)
|
||||
assert not self.uses_mrope, "TPU does not support M-RoPE yet."
|
||||
@@ -354,7 +353,7 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
|
||||
self.mm_budget = (
|
||||
MultiModalBudget(
|
||||
self.renderer_config,
|
||||
self.model_config,
|
||||
self.scheduler_config,
|
||||
self.mm_registry,
|
||||
)
|
||||
@@ -2039,7 +2038,7 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
assert self.mm_budget is not None
|
||||
|
||||
dummy_decoder_data = self.mm_registry.get_decoder_dummy_data(
|
||||
renderer_config=self.renderer_config,
|
||||
model_config=self.model_config,
|
||||
seq_len=self.max_model_len,
|
||||
mm_counts={modality: 1},
|
||||
cache=self.mm_budget.cache,
|
||||
|
||||
+7
-12
@@ -7,7 +7,7 @@ import torch
|
||||
|
||||
from vllm.attention.backends.abstract import AttentionBackend
|
||||
from vllm.attention.layer import Attention
|
||||
from vllm.config import RendererConfig, SchedulerConfig, VllmConfig
|
||||
from vllm.config import ModelConfig, SchedulerConfig, VllmConfig
|
||||
from vllm.model_executor.models.interfaces import MultiModalEmbeddings
|
||||
from vllm.model_executor.models.utils import extract_layer_index
|
||||
from vllm.multimodal.cache import processor_only_cache_from_config
|
||||
@@ -23,29 +23,24 @@ class MultiModalBudget:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
renderer_config: RendererConfig,
|
||||
model_config: ModelConfig,
|
||||
scheduler_config: SchedulerConfig,
|
||||
mm_registry: MultiModalRegistry,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.renderer_config = renderer_config
|
||||
self.model_config = renderer_config.model_config
|
||||
self.model_config = model_config
|
||||
self.scheduler_config = scheduler_config
|
||||
self.mm_registry = mm_registry
|
||||
self.cache = cache = processor_only_cache_from_config(
|
||||
renderer_config, mm_registry
|
||||
)
|
||||
self.cache = cache = processor_only_cache_from_config(model_config, mm_registry)
|
||||
|
||||
self.max_model_len = self.model_config.max_model_len
|
||||
self.max_model_len = model_config.max_model_len
|
||||
self.max_num_reqs = scheduler_config.max_num_seqs
|
||||
|
||||
self.mm_limits = mm_registry.get_mm_limits_per_prompt(
|
||||
renderer_config, cache=cache
|
||||
)
|
||||
self.mm_limits = mm_registry.get_mm_limits_per_prompt(model_config, cache=cache)
|
||||
|
||||
max_tokens_by_modality = mm_registry.get_max_tokens_per_item_by_modality(
|
||||
renderer_config,
|
||||
model_config,
|
||||
cache=cache,
|
||||
profiler_limits=self.mm_limits,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user