Revert "[Renderer] Separate out RendererConfig from ModelConfig (#30145)" (#30199)

This commit is contained in:
Cyrus Leung
2025-12-07 00:00:22 -08:00
committed by GitHub
parent 27f4c2fd46
commit e83b7e379c
105 changed files with 797 additions and 969 deletions
-3
View File
@@ -24,7 +24,6 @@ from vllm.config.multimodal import MultiModalConfig
from vllm.config.observability import ObservabilityConfig
from vllm.config.parallel import EPLBConfig, ParallelConfig
from vllm.config.pooler import PoolerConfig
from vllm.config.renderer import RendererConfig
from vllm.config.scheduler import SchedulerConfig
from vllm.config.speculative import SpeculativeConfig
from vllm.config.speech_to_text import SpeechToTextConfig
@@ -82,8 +81,6 @@ __all__ = [
"ParallelConfig",
# From vllm.config.pooler
"PoolerConfig",
# From vllm.config.renderer
"RendererConfig",
# From vllm.config.scheduler
"SchedulerConfig",
# From vllm.config.speculative
+110 -31
View File
@@ -36,6 +36,7 @@ from vllm.transformers_utils.config import (
uses_xdrope_dim,
)
from vllm.transformers_utils.gguf_utils import (
is_gguf,
is_remote_gguf,
maybe_patch_hf_config_from_gguf,
split_remote_gguf,
@@ -82,6 +83,7 @@ TaskOption = Literal[
"transcription",
"draft",
]
TokenizerMode = Literal["auto", "hf", "slow", "mistral", "deepseek_v32"]
ModelDType = Literal["auto", "half", "float16", "bfloat16", "float", "float32"]
LogprobsMode = Literal[
"raw_logits", "raw_logprobs", "processed_logits", "processed_logprobs"
@@ -129,6 +131,18 @@ class ModelConfig:
Note that the model may support other tasks using the same model runner.
"""
tokenizer: SkipValidation[str] = None # type: ignore
"""Name or path of the Hugging Face tokenizer to use. If unspecified, model
name or path will be used."""
tokenizer_mode: TokenizerMode | str = "auto"
"""Tokenizer mode:\n
- "auto" will use the tokenizer from `mistral_common` for Mistral models
if available, otherwise it will use the "hf" tokenizer.\n
- "hf" will use the fast tokenizer if available.\n
- "slow" will always use the slow tokenizer.\n
- "mistral" will always use the tokenizer from `mistral_common`.\n
- "deepseek_v32" will always use the tokenizer from `deepseek_v32`.\n
- Other custom values can be supported via plugins."""
trust_remote_code: bool = False
"""Trust remote code (e.g., from HuggingFace) when downloading the model
and tokenizer."""
@@ -154,6 +168,13 @@ class ModelConfig:
hf_config_path: str | None = None
"""Name or path of the Hugging Face config to use. If unspecified, model
name or path will be used."""
allowed_local_media_path: str = ""
"""Allowing API requests to read local images or videos from directories
specified by the server file system. This is a security risk. Should only
be enabled in trusted environments."""
allowed_media_domains: list[str] | None = None
"""If set, only media URLs that belong to this domain can be used for
multi-modal inputs. """
revision: str | None = None
"""The specific model version to use. It can be a branch name, a tag name,
or a commit id. If unspecified, will use the default version."""
@@ -161,6 +182,10 @@ class ModelConfig:
"""The specific revision to use for the model code on the Hugging Face Hub.
It can be a branch name, a tag name, or a commit id. If unspecified, will
use the default version."""
tokenizer_revision: str | None = None
"""The specific revision to use for the tokenizer on the Hugging Face Hub.
It can be a branch name, a tag name, or a commit id. If unspecified, will
use the default version."""
max_model_len: SkipValidation[int] = None # type: ignore
"""Model context length (prompt and output). If unspecified, will be
automatically derived from the model config.
@@ -205,6 +230,10 @@ class ModelConfig:
preventing potential numerical issues. Note that even if this is set to
False, cascade attention will be only used when the heuristic tells that
it's beneficial."""
skip_tokenizer_init: bool = False
"""Skip initialization of tokenizer and detokenizer. Expects valid
`prompt_token_ids` and `None` for prompt from the input. The generated
output will contain token ids."""
enable_prompt_embeds: bool = False
"""If `True`, enables passing text embeddings as inputs via the
`prompt_embeds` key.
@@ -265,6 +294,8 @@ class ModelConfig:
logits_processors: list[str | type[LogitsProcessor]] | None = None
"""One or more logits processors' fully-qualified class names or class
definitions"""
io_processor_plugin: str | None = None
"""IOProcessor plugin name to load at model startup"""
# Pooler config
pooler_config: PoolerConfig | None = None
@@ -277,6 +308,7 @@ class ModelConfig:
from the architecture of `self.model`."""
limit_mm_per_prompt: InitVar[dict[str, int | dict[str, int]] | None] = None
enable_mm_embeds: InitVar[bool | None] = None
media_io_kwargs: InitVar[dict[str, dict[str, Any]] | None] = None
mm_processor_kwargs: InitVar[dict[str, Any] | None] = None
mm_processor_cache_gb: InitVar[float | None] = None
mm_processor_cache_type: InitVar[MMCacheType | None] = None
@@ -303,12 +335,18 @@ class ModelConfig:
"runner",
"convert",
"task",
"tokenizer",
"tokenizer_mode",
"seed",
"hf_config_path",
"allowed_local_media_path",
"allowed_media_domains",
"tokenizer_revision",
"spec_target_max_model_len",
"enforce_eager",
"logprobs_mode",
"disable_cascade_attn",
"skip_tokenizer_init",
"served_model_name",
"config_format",
"hf_token",
@@ -316,9 +354,11 @@ class ModelConfig:
"logits_processor_pattern",
"override_attention_dtype",
"logits_processors",
"io_processor_plugin",
"pooler_config",
"multimodal_config",
"limit_mm_per_prompt",
"media_io_kwargs",
"mm_processor_kwargs",
"mm_processor_cache_gb",
"mm_processor_cache_type",
@@ -383,6 +423,7 @@ class ModelConfig:
# Multimodal config init vars
limit_mm_per_prompt: dict[str, int | dict[str, int]] | None,
enable_mm_embeds: bool | None,
media_io_kwargs: dict[str, dict[str, Any]] | None,
mm_processor_kwargs: dict[str, Any] | None,
mm_processor_cache_gb: float | None,
mm_processor_cache_type: MMCacheType | None,
@@ -397,8 +438,13 @@ class ModelConfig:
self.served_model_name = get_served_model_name(
self.model, self.served_model_name
)
self.original_model = self.model
self.model = maybe_model_redirect(self.original_model)
self.model = maybe_model_redirect(self.model)
# The tokenizer is consistent with the model by default.
if self.tokenizer is None:
self.tokenizer = self.model
if self.tokenizer_revision is None:
self.tokenizer_revision = self.revision
self.tokenizer = maybe_model_redirect(self.tokenizer)
if isinstance(self.hf_config_path, str):
self.hf_config_path = maybe_model_redirect(self.hf_config_path)
@@ -419,7 +465,7 @@ class ModelConfig:
hf_overrides_kw[key] = value
hf_overrides_fn = None
self.maybe_pull_model_for_runai(self.model)
self.maybe_pull_model_tokenizer_for_runai(self.model, self.tokenizer)
from vllm.platforms import current_platform
@@ -602,8 +648,7 @@ class ModelConfig:
)
self.original_max_model_len = self.max_model_len
self.recalculate_max_model_len(self.original_max_model_len)
self.max_model_len = self.get_and_verify_max_len(self.max_model_len)
# Init multimodal config if needed
if self._model_info.supports_multimodal:
if (
@@ -619,6 +664,7 @@ class ModelConfig:
mm_config_kwargs = dict(
limit_per_prompt=limit_mm_per_prompt,
enable_mm_embeds=enable_mm_embeds,
media_io_kwargs=media_io_kwargs,
mm_processor_kwargs=mm_processor_kwargs,
mm_processor_cache_gb=mm_processor_cache_gb,
mm_processor_cache_type=mm_processor_cache_type,
@@ -636,8 +682,16 @@ class ModelConfig:
self.multimodal_config = MultiModalConfig(**mm_config_kwargs)
# Multimodal GGUF models must use original repo for mm processing
if is_gguf(self.tokenizer) and self.is_multimodal_model:
raise ValueError(
"Loading a multimodal GGUF model needs to use original "
"tokenizer. Please specify the unquantized hf model's "
"repo name or path using the --tokenizer argument."
)
if self.disable_sliding_window:
# Set after recalculate_max_model_len to ensure that max_model_len
# Set after get_and_verify_max_len to ensure that max_model_len
# can be correctly capped to sliding window size
self.hf_text_config.sliding_window = None
@@ -661,9 +715,10 @@ class ModelConfig:
@model_validator(mode="after")
def validate_model_config_after(self: "ModelConfig") -> "ModelConfig":
if not isinstance(self.tokenizer, str):
raise ValueError("tokenizer must be a string after __post_init__.")
if not isinstance(self.max_model_len, int):
raise ValueError("max_model_len must be an integer after __post_init__.")
return self
def _get_transformers_backend_cls(self) -> str:
@@ -712,17 +767,49 @@ class ModelConfig:
"""The architecture vllm actually used."""
return self._architecture
def maybe_pull_model_for_runai(self, model: str) -> None:
"""Pull model from Object Storage to temporary directory when needed."""
if not is_runai_obj_uri(model):
def maybe_pull_model_tokenizer_for_runai(self, model: str, tokenizer: str) -> None:
"""Pull model/tokenizer from Object Storage to temporary
directory when needed.
Args:
model: Model name or path
tokenizer: Tokenizer name or path
"""
if not (is_runai_obj_uri(model) or is_runai_obj_uri(tokenizer)):
return
object_storage_model = ObjectStorageModel(url=model)
object_storage_model.pull_files(
model, allow_pattern=["*.model", "*.py", "*.json"]
)
self.model_weights = model
self.model = object_storage_model.dir
if is_runai_obj_uri(model):
object_storage_model = ObjectStorageModel(url=model)
object_storage_model.pull_files(
model, allow_pattern=["*.model", "*.py", "*.json"]
)
self.model_weights = model
self.model = object_storage_model.dir
# If tokenizer is same as model, download to same directory
if model == tokenizer:
object_storage_model.pull_files(
model,
ignore_pattern=[
"*.pt",
"*.safetensors",
"*.bin",
"*.tensors",
"*.pth",
],
)
self.tokenizer = object_storage_model.dir
return
# Only download tokenizer if needed and not already handled
if is_runai_obj_uri(tokenizer):
object_storage_tokenizer = ObjectStorageModel(url=tokenizer)
object_storage_tokenizer.pull_files(
model,
ignore_pattern=["*.pt", "*.safetensors", "*.bin", "*.tensors", "*.pth"],
)
self.tokenizer = object_storage_tokenizer.dir
def _get_encoder_config(self):
model = self.model
@@ -1625,38 +1712,30 @@ class ModelConfig:
return dense_modules[-1]["out_features"]
return self.get_hidden_size()
def recalculate_max_model_len(
self,
original_max_model_len: int | None,
*,
tokenizer: str | None = None,
tokenizer_revision: str | None = None,
) -> None:
def get_and_verify_max_len(self, max_model_len: int):
# Consider max_model_len in tokenizer_config only when
# pooling models use absolute position_embedding.
# NOTE: For simplicity we assume `args.model == args.tokenizer`
# since this is
tokenizer_config = None
if (
self.runner_type == "pooling"
and getattr(self.hf_config, "position_embedding_type", "") == "absolute"
):
tokenizer_config = try_get_tokenizer_config(
tokenizer or self.model,
self.tokenizer,
trust_remote_code=self.trust_remote_code,
revision=tokenizer_revision or self.revision,
revision=self.tokenizer_revision,
)
self.max_model_len = _get_and_verify_max_len(
max_model_len = _get_and_verify_max_len(
hf_config=self.hf_text_config,
tokenizer_config=tokenizer_config,
max_model_len=original_max_model_len,
max_model_len=max_model_len,
disable_sliding_window=self.disable_sliding_window,
sliding_window=self.get_sliding_window(),
spec_target_max_model_len=self.spec_target_max_model_len,
encoder_config=self.encoder_config,
)
logger.info("Using max model len %s", self.max_model_len)
logger.info("Using max model len %s", max_model_len)
return max_model_len
@property
def attn_type(self) -> AttnTypeStr:
+4
View File
@@ -79,6 +79,10 @@ class MultiModalConfig:
WARNING: The vLLM engine may crash if incorrect shape of embeddings is passed.
Only enable this flag for trusted users!"""
media_io_kwargs: dict[str, dict[str, Any]] = Field(default_factory=dict)
"""Additional args passed to process media inputs, keyed by modalities.
For example, to set num_frames for video, set
`--media-io-kwargs '{"video": {"num_frames": 40} }'`"""
mm_processor_kwargs: dict[str, object] | None = None
"""Arguments to be forwarded to the model's processor for multi-modal data,
e.g., image processor. Overrides for the multi-modal processor obtained
-109
View File
@@ -1,109 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any, Literal
from pydantic import Field, SkipValidation
from pydantic.dataclasses import dataclass
from vllm.config.model import ModelConfig
from vllm.config.utils import config
from vllm.transformers_utils.gguf_utils import is_gguf
from vllm.transformers_utils.runai_utils import ObjectStorageModel, is_runai_obj_uri
from vllm.transformers_utils.utils import maybe_model_redirect
TokenizerMode = Literal["auto", "hf", "slow", "mistral", "deepseek_v32"]
@config
@dataclass
class RendererConfig:
"""Configuration for the renderer."""
# NOTE: In reality, this is a required argument.
# We provide a dummy default value here to generate the CLI args.
model_config: SkipValidation[ModelConfig] = None # type: ignore
"""Provides model context to the renderer."""
tokenizer: str = ""
"""Name or path of the Hugging Face tokenizer to use. If unspecified, model
name or path will be used."""
tokenizer_mode: TokenizerMode | str = "auto"
"""Tokenizer mode:\n
- "auto" will use the tokenizer from `mistral_common` for Mistral models
if available, otherwise it will use the "hf" tokenizer.\n
- "hf" will use the fast tokenizer if available.\n
- "slow" will always use the slow tokenizer.\n
- "mistral" will always use the tokenizer from `mistral_common`.\n
- "deepseek_v32" will always use the tokenizer from `deepseek_v32`.\n
- Other custom values can be supported via plugins."""
tokenizer_revision: str | None = None
"""The specific revision to use for the tokenizer on the Hugging Face Hub.
It can be a branch name, a tag name, or a commit id. If unspecified, will
use the default version."""
skip_tokenizer_init: bool = False
"""Skip initialization of tokenizer and detokenizer. Expects valid
`prompt_token_ids` and `None` for prompt from the input. The generated
output will contain token ids."""
io_processor_plugin: str | None = None
"""IOProcessor plugin name to load at model startup."""
media_io_kwargs: dict[str, dict[str, Any]] = Field(default_factory=dict)
"""Additional args passed to process media inputs, keyed by modalities.
For example, to set num_frames for video, set
`--media-io-kwargs '{"video": {"num_frames": 40} }'`"""
allowed_local_media_path: str = ""
"""Allowing API requests to read local images or videos from directories
specified by the server file system. This is a security risk. Should only
be enabled in trusted environments."""
allowed_media_domains: list[str] | None = None
"""If set, only media URLs that belong to this domain can be used for
multi-modal inputs. """
@property
def trust_remote_code(self) -> bool:
return self.model_config.trust_remote_code
def __post_init__(self) -> None:
model_config = self.model_config
# The tokenizer is consistent with the model by default.
if not self.tokenizer:
self.tokenizer = (
ModelConfig.model
if model_config is None
else model_config.original_model
)
if not self.tokenizer_revision:
self.tokenizer_revision = (
ModelConfig.revision if model_config is None else model_config.revision
)
self.original_tokenizer = self.tokenizer
self.tokenizer = maybe_model_redirect(self.original_tokenizer)
self.maybe_pull_tokenizer_for_runai(self.tokenizer)
# Multimodal GGUF models must use original repo for mm processing
is_multimodal_model = (
ModelConfig.is_multimodal_model
if model_config is None
else model_config.is_multimodal_model
)
if is_gguf(self.tokenizer) and is_multimodal_model:
raise ValueError(
"Loading a multimodal GGUF model needs to use original "
"tokenizer. Please specify the unquantized hf model's "
"repo name or path using the --tokenizer argument."
)
def maybe_pull_tokenizer_for_runai(self, tokenizer: str) -> None:
"""Pull tokenizer from Object Storage to temporary directory when needed."""
if not is_runai_obj_uri(tokenizer):
return
object_storage_tokenizer = ObjectStorageModel(url=tokenizer)
object_storage_tokenizer.pull_files(
tokenizer,
ignore_pattern=["*.pt", "*.safetensors", "*.bin", "*.tensors", "*.pth"],
)
self.tokenizer = object_storage_tokenizer.dir
+5
View File
@@ -322,11 +322,16 @@ class SpeculativeConfig:
self.draft_model_config = ModelConfig(
model=self.model,
runner="draft",
tokenizer=self.target_model_config.tokenizer,
tokenizer_mode=self.target_model_config.tokenizer_mode,
trust_remote_code=self.target_model_config.trust_remote_code,
allowed_local_media_path=self.target_model_config.allowed_local_media_path,
allowed_media_domains=self.target_model_config.allowed_media_domains,
dtype=self.target_model_config.dtype,
seed=self.target_model_config.seed,
revision=self.revision,
code_revision=self.code_revision,
tokenizer_revision=self.target_model_config.tokenizer_revision,
spec_target_max_model_len=self.target_model_config.max_model_len,
quantization=self.quantization,
enforce_eager=self.target_model_config.enforce_eager,
+10 -15
View File
@@ -39,7 +39,6 @@ from .lora import LoRAConfig
from .model import ModelConfig
from .observability import ObservabilityConfig
from .parallel import ParallelConfig
from .renderer import RendererConfig
from .scheduler import SchedulerConfig
from .speculative import SpeculativeConfig
from .structured_outputs import StructuredOutputsConfig
@@ -182,8 +181,6 @@ class VllmConfig:
# try to download a model
model_config: ModelConfig = Field(default=None)
"""Model configuration."""
renderer_config: RendererConfig = Field(default_factory=RendererConfig)
"""Renderer configuration."""
cache_config: CacheConfig = Field(default_factory=CacheConfig)
"""Cache configuration."""
parallel_config: ParallelConfig = Field(default_factory=ParallelConfig)
@@ -744,7 +741,7 @@ class VllmConfig:
from vllm.multimodal import MULTIMODAL_REGISTRY
self.scheduler_config.max_num_encoder_input_tokens = (
MULTIMODAL_REGISTRY.get_encdec_max_encoder_len(self.renderer_config)
MULTIMODAL_REGISTRY.get_encdec_max_encoder_len(self.model_config)
)
logger.debug(
"Encoder-decoder model detected: setting "
@@ -1189,13 +1186,11 @@ class VllmConfig:
computed_compile_ranges_split_points
)
def recalculate_max_model_len(self, original_max_model_len: int | None) -> None:
# Can only be called during try_verify_and_update_config
self.model_config.recalculate_max_model_len(
original_max_model_len,
tokenizer=self.renderer_config.tokenizer,
tokenizer_revision=self.renderer_config.tokenizer_revision,
)
def recalculate_max_model_len(self, max_model_len: int):
# Can only be called in try_verify_and_update_config
model_config = self.model_config
max_model_len = model_config.get_and_verify_max_len(max_model_len)
self.model_config.max_model_len = max_model_len
def try_verify_and_update_config(self):
if self.model_config is None:
@@ -1269,11 +1264,11 @@ class VllmConfig:
return (
f"model={self.model_config.model!r}, "
f"speculative_config={self.speculative_config!r}, "
f"tokenizer={self.renderer_config.tokenizer!r}, "
f"skip_tokenizer_init={self.renderer_config.skip_tokenizer_init}, "
f"tokenizer_mode={self.renderer_config.tokenizer_mode}, "
f"tokenizer={self.model_config.tokenizer!r}, "
f"skip_tokenizer_init={self.model_config.skip_tokenizer_init}, "
f"tokenizer_mode={self.model_config.tokenizer_mode}, "
f"revision={self.model_config.revision}, "
f"tokenizer_revision={self.renderer_config.tokenizer_revision}, "
f"tokenizer_revision={self.model_config.tokenizer_revision}, "
f"trust_remote_code={self.model_config.trust_remote_code}, "
f"dtype={self.model_config.dtype}, "
f"max_seq_len={self.model_config.max_model_len}, "
+40 -59
View File
@@ -71,11 +71,11 @@ from vllm.config.model import (
ModelDType,
RunnerOption,
TaskOption,
TokenizerMode,
)
from vllm.config.multimodal import MMCacheType, MMEncoderTPMode
from vllm.config.observability import DetailedTraceModules
from vllm.config.parallel import DistributedExecutorBackend, ExpertPlacementStrategy
from vllm.config.renderer import RendererConfig, TokenizerMode
from vllm.config.scheduler import SchedulerPolicy
from vllm.config.utils import get_field
from vllm.config.vllm import OptimizationLevel
@@ -355,12 +355,17 @@ class EngineArgs:
model: str = ModelConfig.model
served_model_name: str | list[str] | None = ModelConfig.served_model_name
tokenizer: str | None = ModelConfig.tokenizer
hf_config_path: str | None = ModelConfig.hf_config_path
runner: RunnerOption = ModelConfig.runner
convert: ConvertOption = ModelConfig.convert
task: TaskOption | None = ModelConfig.task
skip_tokenizer_init: bool = ModelConfig.skip_tokenizer_init
enable_prompt_embeds: bool = ModelConfig.enable_prompt_embeds
tokenizer_mode: TokenizerMode | str = ModelConfig.tokenizer_mode
trust_remote_code: bool = ModelConfig.trust_remote_code
allowed_local_media_path: str = ModelConfig.allowed_local_media_path
allowed_media_domains: list[str] | None = ModelConfig.allowed_media_domains
download_dir: str | None = LoadConfig.download_dir
safetensors_load_strategy: str = LoadConfig.safetensors_load_strategy
load_format: str | LoadFormats = LoadConfig.load_format
@@ -444,6 +449,7 @@ class EngineArgs:
code_revision: str | None = ModelConfig.code_revision
hf_token: bool | str | None = ModelConfig.hf_token
hf_overrides: HfOverrides = get_field(ModelConfig, "hf_overrides")
tokenizer_revision: str | None = ModelConfig.tokenizer_revision
quantization: QuantizationMethods | None = ModelConfig.quantization
enforce_eager: bool = ModelConfig.enforce_eager
disable_custom_all_reduce: bool = ParallelConfig.disable_custom_all_reduce
@@ -452,6 +458,9 @@ class EngineArgs:
)
enable_mm_embeds: bool = MultiModalConfig.enable_mm_embeds
interleave_mm_strings: bool = MultiModalConfig.interleave_mm_strings
media_io_kwargs: dict[str, dict[str, Any]] = get_field(
MultiModalConfig, "media_io_kwargs"
)
mm_processor_kwargs: dict[str, Any] | None = MultiModalConfig.mm_processor_kwargs
disable_mm_preprocessor_cache: bool = False # DEPRECATED
mm_processor_cache_gb: float = MultiModalConfig.mm_processor_cache_gb
@@ -465,19 +474,9 @@ class EngineArgs:
mm_encoder_attn_backend: AttentionBackendEnum | str | None = (
MultiModalConfig.mm_encoder_attn_backend
)
io_processor_plugin: str | None = None
skip_mm_profiling: bool = MultiModalConfig.skip_mm_profiling
video_pruning_rate: float = MultiModalConfig.video_pruning_rate
# Renderer fields
tokenizer: str | None = None
tokenizer_mode: TokenizerMode | str = RendererConfig.tokenizer_mode
tokenizer_revision: str | None = RendererConfig.tokenizer_revision
skip_tokenizer_init: bool = RendererConfig.skip_tokenizer_init
io_processor_plugin: str | None = None
media_io_kwargs: dict[str, dict[str, Any]] = get_field(
RendererConfig, "media_io_kwargs"
)
allowed_local_media_path: str = RendererConfig.allowed_local_media_path
allowed_media_domains: list[str] | None = RendererConfig.allowed_media_domains
# LoRA fields
enable_lora: bool = False
max_loras: int = LoRAConfig.max_loras
@@ -628,14 +627,25 @@ class EngineArgs:
model_group.add_argument("--runner", **model_kwargs["runner"])
model_group.add_argument("--convert", **model_kwargs["convert"])
model_group.add_argument("--task", **model_kwargs["task"], deprecated=True)
model_group.add_argument("--tokenizer", **model_kwargs["tokenizer"])
model_group.add_argument("--tokenizer-mode", **model_kwargs["tokenizer_mode"])
model_group.add_argument(
"--trust-remote-code", **model_kwargs["trust_remote_code"]
)
model_group.add_argument("--dtype", **model_kwargs["dtype"])
model_group.add_argument("--seed", **model_kwargs["seed"])
model_group.add_argument("--hf-config-path", **model_kwargs["hf_config_path"])
model_group.add_argument(
"--allowed-local-media-path", **model_kwargs["allowed_local_media_path"]
)
model_group.add_argument(
"--allowed-media-domains", **model_kwargs["allowed_media_domains"]
)
model_group.add_argument("--revision", **model_kwargs["revision"])
model_group.add_argument("--code-revision", **model_kwargs["code_revision"])
model_group.add_argument(
"--tokenizer-revision", **model_kwargs["tokenizer_revision"]
)
model_group.add_argument("--max-model-len", **model_kwargs["max_model_len"])
model_group.add_argument("--quantization", "-q", **model_kwargs["quantization"])
model_group.add_argument("--enforce-eager", **model_kwargs["enforce_eager"])
@@ -647,6 +657,9 @@ class EngineArgs:
model_group.add_argument(
"--disable-cascade-attn", **model_kwargs["disable_cascade_attn"]
)
model_group.add_argument(
"--skip-tokenizer-init", **model_kwargs["skip_tokenizer_init"]
)
model_group.add_argument(
"--enable-prompt-embeds", **model_kwargs["enable_prompt_embeds"]
)
@@ -685,34 +698,8 @@ class EngineArgs:
model_group.add_argument(
"--logits-processors", **model_kwargs["logits_processors"]
)
# Renderer arguments
renderer_kwargs = get_kwargs(RendererConfig)
renderer_group = parser.add_argument_group(
title="RendererConfig",
description=RendererConfig.__doc__,
)
renderer_group.add_argument("--tokenizer", **renderer_kwargs["tokenizer"])
renderer_group.add_argument(
"--tokenizer-mode", **renderer_kwargs["tokenizer_mode"]
)
renderer_group.add_argument(
"--tokenizer-revision", **renderer_kwargs["tokenizer_revision"]
)
renderer_group.add_argument(
"--skip-tokenizer-init", **renderer_kwargs["skip_tokenizer_init"]
)
renderer_group.add_argument(
"--media-io-kwargs", **renderer_kwargs["media_io_kwargs"]
)
renderer_group.add_argument(
"--allowed-local-media-path", **renderer_kwargs["allowed_local_media_path"]
)
renderer_group.add_argument(
"--allowed-media-domains", **renderer_kwargs["allowed_media_domains"]
)
renderer_group.add_argument(
"--io-processor-plugin", **renderer_kwargs["io_processor_plugin"]
model_group.add_argument(
"--io-processor-plugin", **model_kwargs["io_processor_plugin"]
)
# Model loading arguments
@@ -962,6 +949,9 @@ class EngineArgs:
multimodal_group.add_argument(
"--enable-mm-embeds", **multimodal_kwargs["enable_mm_embeds"]
)
multimodal_group.add_argument(
"--media-io-kwargs", **multimodal_kwargs["media_io_kwargs"]
)
multimodal_group.add_argument(
"--mm-processor-kwargs", **multimodal_kwargs["mm_processor_kwargs"]
)
@@ -1265,13 +1255,18 @@ class EngineArgs:
runner=self.runner,
convert=self.convert,
task=self.task,
tokenizer=self.tokenizer,
tokenizer_mode=self.tokenizer_mode,
trust_remote_code=self.trust_remote_code,
allowed_local_media_path=self.allowed_local_media_path,
allowed_media_domains=self.allowed_media_domains,
dtype=self.dtype,
seed=self.seed,
revision=self.revision,
code_revision=self.code_revision,
hf_token=self.hf_token,
hf_overrides=self.hf_overrides,
tokenizer_revision=self.tokenizer_revision,
max_model_len=self.max_model_len,
quantization=self.quantization,
enforce_eager=self.enforce_eager,
@@ -1279,11 +1274,13 @@ class EngineArgs:
logprobs_mode=self.logprobs_mode,
disable_sliding_window=self.disable_sliding_window,
disable_cascade_attn=self.disable_cascade_attn,
skip_tokenizer_init=self.skip_tokenizer_init,
enable_prompt_embeds=self.enable_prompt_embeds,
served_model_name=self.served_model_name,
limit_mm_per_prompt=self.limit_mm_per_prompt,
enable_mm_embeds=self.enable_mm_embeds,
interleave_mm_strings=self.interleave_mm_strings,
media_io_kwargs=self.media_io_kwargs,
skip_mm_profiling=self.skip_mm_profiling,
config_format=self.config_format,
mm_processor_kwargs=self.mm_processor_kwargs,
@@ -1301,6 +1298,7 @@ class EngineArgs:
override_attention_dtype=self.override_attention_dtype,
logits_processors=self.logits_processors,
video_pruning_rate=self.video_pruning_rate,
io_processor_plugin=self.io_processor_plugin,
)
def validate_tensorizer_args(self):
@@ -1396,25 +1394,9 @@ class EngineArgs:
)
model_config = self.create_model_config()
renderer_config = RendererConfig(
model_config=model_config,
tokenizer=self.tokenizer or "",
tokenizer_mode=self.tokenizer_mode,
tokenizer_revision=self.tokenizer_revision,
skip_tokenizer_init=self.skip_tokenizer_init,
io_processor_plugin=self.io_processor_plugin,
media_io_kwargs=self.media_io_kwargs,
allowed_local_media_path=self.allowed_local_media_path,
allowed_media_domains=self.allowed_media_domains,
)
model_config.recalculate_max_model_len(
model_config.original_max_model_len,
tokenizer=renderer_config.tokenizer,
tokenizer_revision=renderer_config.tokenizer_revision,
)
self.model = model_config.model
self.tokenizer = model_config.tokenizer
self._check_feature_supported(model_config)
self._set_default_chunked_prefill_and_prefix_caching_args(model_config)
self._set_default_max_num_seqs_and_batched_tokens_args(
@@ -1786,7 +1768,6 @@ class EngineArgs:
)
config = VllmConfig(
model_config=model_config,
renderer_config=renderer_config,
cache_config=cache_config,
parallel_config=parallel_config,
scheduler_config=scheduler_config,
+1 -2
View File
@@ -5,7 +5,7 @@ from abc import ABC, abstractmethod
from collections.abc import AsyncGenerator, Iterable, Mapping
from typing import Any
from vllm.config import ModelConfig, RendererConfig, VllmConfig
from vllm.config import ModelConfig, VllmConfig
from vllm.inputs.data import PromptType
from vllm.lora.request import LoRARequest
from vllm.outputs import PoolingRequestOutput, RequestOutput
@@ -22,7 +22,6 @@ class EngineClient(ABC):
"""Protocol class for Clients to Engine"""
vllm_config: VllmConfig
renderer_config: RendererConfig
model_config: ModelConfig
input_processor: InputProcessor
io_processor: IOProcessor | None
+34 -45
View File
@@ -44,7 +44,7 @@ from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast, Processor
from typing_extensions import Required, TypedDict
from vllm import envs
from vllm.config import ModelConfig, RendererConfig
from vllm.config import ModelConfig
from vllm.logger import init_logger
from vllm.model_executor.models import SupportsMultiModal
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict, MultiModalUUIDDict
@@ -452,10 +452,9 @@ This is needed because `lru_cache` does not cache when an exception happens.
def _try_get_processor_chat_template(
tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast,
*,
trust_remote_code: bool,
model_config: ModelConfig,
) -> str | None:
cache_key = (tokenizer.name_or_path, trust_remote_code)
cache_key = (tokenizer.name_or_path, model_config.trust_remote_code)
if cache_key in _PROCESSOR_CHAT_TEMPLATES:
return _PROCESSOR_CHAT_TEMPLATES[cache_key]
@@ -467,7 +466,7 @@ def _try_get_processor_chat_template(
PreTrainedTokenizerFast,
ProcessorMixin,
),
trust_remote_code=trust_remote_code,
trust_remote_code=model_config.trust_remote_code,
)
if (
isinstance(processor, ProcessorMixin)
@@ -500,10 +499,7 @@ def resolve_hf_chat_template(
# 2nd priority: AutoProcessor chat template, unless tool calling is enabled
if tools is None:
chat_template = _try_get_processor_chat_template(
tokenizer,
trust_remote_code=model_config.trust_remote_code,
)
chat_template = _try_get_processor_chat_template(tokenizer, model_config)
if chat_template is not None:
return chat_template
@@ -517,10 +513,10 @@ def resolve_hf_chat_template(
exc_info=True,
)
# 4th priority: Predefined fallbacks]
# 4th priority: Predefined fallbacks
path = get_chat_template_fallback_path(
model_type=model_config.hf_config.model_type,
tokenizer_name_or_path=tokenizer.name_or_path,
tokenizer_name_or_path=model_config.tokenizer,
)
if path is not None:
logger.info_once(
@@ -542,14 +538,14 @@ def _resolve_chat_template_content_format(
tools: list[dict[str, Any]] | None,
tokenizer: TokenizerLike | None,
*,
renderer_config: RendererConfig,
model_config: ModelConfig,
) -> _ChatTemplateContentFormat:
if isinstance(tokenizer, (PreTrainedTokenizer, PreTrainedTokenizerFast)):
hf_chat_template = resolve_hf_chat_template(
tokenizer,
chat_template=chat_template,
tools=tools,
model_config=renderer_config.model_config,
model_config=model_config,
)
else:
hf_chat_template = None
@@ -599,7 +595,7 @@ def resolve_chat_template_content_format(
given_format: ChatTemplateContentFormatOption,
tokenizer: TokenizerLike | None,
*,
renderer_config: RendererConfig,
model_config: ModelConfig,
) -> _ChatTemplateContentFormat:
if given_format != "auto":
return given_format
@@ -608,7 +604,7 @@ def resolve_chat_template_content_format(
chat_template,
tools,
tokenizer,
renderer_config=renderer_config,
model_config=model_config,
)
_log_chat_template_content_format(
@@ -631,32 +627,32 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
maximum per prompt.
"""
def __init__(self, renderer_config: RendererConfig):
def __init__(self, model_config: ModelConfig):
super().__init__()
self._renderer_config = renderer_config
self._model_config = model_config
self._items_by_modality = defaultdict[str, list[_T | None]](list)
self._uuids_by_modality = defaultdict[str, list[str | None]](list)
@property
def renderer_config(self) -> RendererConfig:
return self._renderer_config
def model_config(self) -> ModelConfig:
return self._model_config
@cached_property
def model_cls(self) -> type[SupportsMultiModal]:
from vllm.model_executor.model_loader import get_model_cls
model_cls = get_model_cls(self.renderer_config.model_config)
model_cls = get_model_cls(self.model_config)
return cast(type[SupportsMultiModal], model_cls)
@property
def allowed_local_media_path(self):
return self._renderer_config.allowed_local_media_path
return self._model_config.allowed_local_media_path
@property
def allowed_media_domains(self):
return self._renderer_config.allowed_media_domains
return self._model_config.allowed_media_domains
@property
def mm_registry(self):
@@ -664,7 +660,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
@cached_property
def mm_processor(self):
return self.mm_registry.create_processor(self.renderer_config)
return self.mm_registry.create_processor(self.model_config)
def add(
self,
@@ -855,20 +851,19 @@ class MultiModalContentParser(BaseMultiModalContentParser):
super().__init__()
self._tracker = tracker
multimodal_config = self._tracker.model_config.multimodal_config
media_io_kwargs = getattr(multimodal_config, "media_io_kwargs", None)
self._connector: MediaConnector = MEDIA_CONNECTOR_REGISTRY.load(
envs.VLLM_MEDIA_CONNECTOR,
media_io_kwargs=self.renderer_config.media_io_kwargs,
media_io_kwargs=media_io_kwargs,
allowed_local_media_path=tracker.allowed_local_media_path,
allowed_media_domains=tracker.allowed_media_domains,
)
@property
def renderer_config(self) -> RendererConfig:
return self._tracker.renderer_config
@property
def model_config(self) -> ModelConfig:
return self.renderer_config.model_config
return self._tracker.model_config
def parse_image(self, image_url: str | None, uuid: str | None = None) -> None:
image = self._connector.fetch_image(image_url) if image_url else None
@@ -968,20 +963,18 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
super().__init__()
self._tracker = tracker
multimodal_config = self._tracker.model_config.multimodal_config
media_io_kwargs = getattr(multimodal_config, "media_io_kwargs", None)
self._connector: MediaConnector = MEDIA_CONNECTOR_REGISTRY.load(
envs.VLLM_MEDIA_CONNECTOR,
media_io_kwargs=self.renderer_config.media_io_kwargs,
media_io_kwargs=media_io_kwargs,
allowed_local_media_path=tracker.allowed_local_media_path,
allowed_media_domains=tracker.allowed_media_domains,
)
@property
def renderer_config(self) -> RendererConfig:
return self._tracker.renderer_config
@property
def model_config(self) -> ModelConfig:
return self.renderer_config.model_config
return self._tracker.model_config
def parse_image(self, image_url: str | None, uuid: str | None = None) -> None:
image_coro = self._connector.fetch_image_async(image_url) if image_url else None
@@ -1611,17 +1604,15 @@ def _postprocess_messages(messages: list[ConversationMessage]) -> None:
def parse_chat_messages(
messages: list[ChatCompletionMessageParam],
renderer_config: RendererConfig,
model_config: ModelConfig,
content_format: _ChatTemplateContentFormat,
) -> tuple[
list[ConversationMessage],
MultiModalDataDict | None,
MultiModalUUIDDict | None,
]:
model_config = renderer_config.model_config
conversation: list[ConversationMessage] = []
mm_tracker = MultiModalItemTracker(renderer_config)
mm_tracker = MultiModalItemTracker(model_config)
for msg in messages:
sub_messages = _parse_chat_message_content(
@@ -1644,17 +1635,15 @@ def parse_chat_messages(
def parse_chat_messages_futures(
messages: list[ChatCompletionMessageParam],
renderer_config: RendererConfig,
model_config: ModelConfig,
content_format: _ChatTemplateContentFormat,
) -> tuple[
list[ConversationMessage],
Awaitable[MultiModalDataDict | None],
MultiModalUUIDDict | None,
]:
model_config = renderer_config.model_config
conversation: list[ConversationMessage] = []
mm_tracker = AsyncMultiModalItemTracker(renderer_config)
mm_tracker = AsyncMultiModalItemTracker(model_config)
for msg in messages:
sub_messages = _parse_chat_message_content(
@@ -1759,14 +1748,14 @@ def apply_hf_chat_template(
chat_template: str | None,
tools: list[dict[str, Any]] | None,
*,
renderer_config: RendererConfig,
model_config: ModelConfig,
**kwargs: Any,
) -> str:
hf_chat_template = resolve_hf_chat_template(
tokenizer,
chat_template=chat_template,
tools=tools,
model_config=renderer_config.model_config,
model_config=model_config,
)
if hf_chat_template is None:
+6 -8
View File
@@ -29,8 +29,8 @@ from vllm.config.model import (
HfOverrides,
ModelDType,
RunnerOption,
TokenizerMode,
)
from vllm.config.renderer import TokenizerMode
from vllm.engine.arg_utils import EngineArgs
from vllm.entrypoints.chat_utils import (
ChatCompletionMessageParam,
@@ -343,7 +343,6 @@ class LLM:
logger.info("Supported tasks: %s", supported_tasks)
self.supported_tasks = supported_tasks
self.renderer_config = self.llm_engine.renderer_config
self.model_config = self.llm_engine.model_config
self.input_processor = self.llm_engine.input_processor
self.io_processor = self.llm_engine.io_processor
@@ -809,13 +808,13 @@ class LLM:
list_of_messages = [cast(list[ChatCompletionMessageParam], messages)]
tokenizer = self.get_tokenizer()
renderer_config = self.renderer_config
model_config = self.model_config
resolved_content_format = resolve_chat_template_content_format(
chat_template,
tools,
chat_template_content_format,
tokenizer,
renderer_config=renderer_config,
model_config=model_config,
)
_chat_template_kwargs: dict[str, Any] = dict(
@@ -834,7 +833,7 @@ class LLM:
# the chat message parsing for it.
conversation, mm_data, mm_uuids = parse_chat_messages(
msgs,
renderer_config,
model_config,
content_format=resolved_content_format,
)
@@ -848,7 +847,7 @@ class LLM:
prompt_str = apply_hf_chat_template(
tokenizer=tokenizer,
conversation=conversation,
renderer_config=renderer_config,
model_config=model_config,
**_chat_template_kwargs,
)
# Special tokens are already included in chat templates so
@@ -1291,7 +1290,6 @@ class LLM:
lora_request: list[LoRARequest] | LoRARequest | None = None,
tokenization_kwargs: dict[str, Any] | None = None,
) -> list[ScoringRequestOutput]:
renderer_config = self.renderer_config
model_config = self.model_config
if isinstance(tokenizer, MistralTokenizer):
@@ -1319,7 +1317,7 @@ class LLM:
for q, d in input_pairs:
_, engine_prompt = get_score_prompt(
renderer_config=renderer_config,
model_config=model_config,
data_1=q,
data_2=d,
tokenizer=tokenizer,
+1 -1
View File
@@ -1099,7 +1099,7 @@ async def init_app_state(
logger.info("Supported tasks: %s", supported_tasks)
resolved_chat_template = await process_chat_template(
args.chat_template, engine_client, vllm_config.renderer_config
args.chat_template, engine_client, vllm_config.model_config
)
if args.tool_server == "demo":
@@ -122,7 +122,7 @@ class OpenAIServingCompletion(OpenAIServing):
try:
lora_request = self._maybe_get_adapters(request)
if self.renderer_config.skip_tokenizer_init:
if self.model_config.skip_tokenizer_init:
tokenizer = None
else:
tokenizer = await self.engine_client.get_tokenizer()
+5 -6
View File
@@ -291,7 +291,6 @@ class OpenAIServing:
self.input_processor = self.models.input_processor
self.io_processor = self.models.io_processor
self.renderer_config = self.models.renderer_config
self.model_config = self.models.model_config
self.max_model_len = self.model_config.max_model_len
@@ -1101,18 +1100,18 @@ class OpenAIServing:
Sequence[RequestPrompt],
list[EngineTokensPrompt],
]:
renderer_config = self.renderer_config
model_config = self.model_config
resolved_content_format = resolve_chat_template_content_format(
chat_template,
tool_dicts,
chat_template_content_format,
tokenizer,
renderer_config=renderer_config,
model_config=model_config,
)
conversation, mm_data_future, mm_uuids = parse_chat_messages_futures(
messages,
renderer_config,
model_config,
content_format=resolved_content_format,
)
@@ -1139,14 +1138,14 @@ class OpenAIServing:
request_prompt = tokenizer.apply_chat_template(
conversation=conversation,
messages=messages,
model_config=renderer_config.model_config,
model_config=model_config,
**_chat_template_kwargs,
)
else:
request_prompt = apply_hf_chat_template(
tokenizer=tokenizer,
conversation=conversation,
renderer_config=renderer_config,
model_config=model_config,
**_chat_template_kwargs,
)
@@ -71,7 +71,6 @@ class OpenAIServingModels:
self.input_processor = self.engine_client.input_processor
self.io_processor = self.engine_client.io_processor
self.renderer_config = self.engine_client.renderer_config
self.model_config = self.engine_client.model_config
self.max_model_len = self.model_config.max_model_len
+5 -5
View File
@@ -91,7 +91,7 @@ class OpenAISpeechToText(OpenAIServing):
self.task_type = task_type
self.asr_config = self.model_cls.get_speech_to_text_config(
self.renderer_config, task_type
self.model_config, task_type
)
self.enable_force_include_usage = enable_force_include_usage
@@ -101,8 +101,8 @@ class OpenAISpeechToText(OpenAIServing):
self.tokenizer = cast(
PreTrainedTokenizerBase,
get_tokenizer(
tokenizer_name=self.renderer_config.tokenizer,
tokenizer_mode=self.renderer_config.tokenizer_mode,
tokenizer_name=self.model_config.tokenizer,
tokenizer_mode=self.model_config.tokenizer_mode,
),
)
@@ -154,7 +154,7 @@ class OpenAISpeechToText(OpenAIServing):
prompt = self.model_cls.get_generation_prompt(
audio=chunk,
stt_config=self.asr_config,
renderer_config=self.renderer_config,
model_config=self.model_config,
language=language,
task_type=self.task_type,
request_prompt=request.prompt,
@@ -428,7 +428,7 @@ class OpenAISpeechToText(OpenAIServing):
if res.prompt_token_ids is not None:
num_prompt_tokens = len(res.prompt_token_ids)
if audio_tokens := self.model_cls.get_num_audio_tokens(
audio_duration_s, self.asr_config, self.renderer_config
audio_duration_s, self.asr_config, self.model_config
):
num_prompt_tokens += audio_tokens
+1 -1
View File
@@ -94,7 +94,7 @@ class OpenAIServingPooling(OpenAIServing):
try:
lora_request = self._maybe_get_adapters(request)
if self.renderer_config.skip_tokenizer_init:
if self.model_config.skip_tokenizer_init:
tokenizer = None
else:
tokenizer = await self.engine_client.get_tokenizer()
+3 -1
View File
@@ -160,8 +160,10 @@ class ServingScores(OpenAIServing):
data_1: str | ScoreContentPartParam,
data_2: str | ScoreContentPartParam,
) -> tuple[str, TokensPrompt]:
model_config = self.model_config
full_prompt, engine_prompt = get_score_prompt(
renderer_config=self.renderer_config,
model_config=model_config,
data_1=data_1,
data_2=data_2,
tokenizer=tokenizer,
+5 -8
View File
@@ -5,7 +5,7 @@ from typing import Any, TypeAlias, cast
from torch.nn import CosineSimilarity
from typing_extensions import Required, TypedDict
from vllm.config import ModelConfig, RendererConfig
from vllm.config import ModelConfig
from vllm.entrypoints.chat_utils import (
BaseMultiModalItemTracker,
ChatCompletionContentPartImageEmbedsParam,
@@ -88,9 +88,9 @@ def _validate_score_input_lens(
def parse_score_data(
data_1: str | ScoreContentPartParam,
data_2: str | ScoreContentPartParam,
renderer_config: RendererConfig,
model_config: ModelConfig,
) -> tuple[str, str, MultiModalDataDict | None]:
mm_tracker = MultiModalItemTracker(renderer_config)
mm_tracker = MultiModalItemTracker(model_config)
content_1 = _parse_score_content(data_1, mm_tracker)
content_2 = _parse_score_content(data_2, mm_tracker)
@@ -176,7 +176,7 @@ def post_process_tokens(
def get_score_prompt(
renderer_config: RendererConfig,
model_config: ModelConfig,
tokenizer: TokenizerLike,
tokenization_kwargs: dict[str, Any],
data_1: str | ScoreContentPartParam,
@@ -185,14 +185,11 @@ def get_score_prompt(
prompt_1, prompt_2, mm_data = parse_score_data(
data_1,
data_2,
renderer_config,
model_config,
)
from vllm.model_executor.model_loader import get_model_cls
model_config = renderer_config.model_config
model = get_model_cls(model_config)
if supports_score_template(model):
full_prompt = apply_score_template(model_config, prompt_1, prompt_2)
prompt_inputs = tokenizer(full_prompt, **tokenization_kwargs)
+4 -4
View File
@@ -13,7 +13,7 @@ from fastapi import Request
from fastapi.responses import JSONResponse, StreamingResponse
from starlette.background import BackgroundTask, BackgroundTasks
from vllm.config import RendererConfig
from vllm.config import ModelConfig
from vllm.engine.arg_utils import EngineArgs
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.chat_utils import (
@@ -288,7 +288,7 @@ def process_lora_modules(
async def process_chat_template(
args_chat_template: Path | str | None,
engine_client: EngineClient,
renderer_config: RendererConfig,
model_config: ModelConfig,
) -> str | None:
resolved_chat_template = load_chat_template(args_chat_template)
if resolved_chat_template is not None:
@@ -305,7 +305,7 @@ async def process_chat_template(
tokenizer=tokenizer,
chat_template=None,
tools=None,
model_config=renderer_config.model_config,
model_config=model_config,
)
if hf_chat_template != resolved_chat_template:
@@ -314,6 +314,6 @@ async def process_chat_template(
"It is different from official chat template '%s'. "
"This discrepancy may lead to performance degradation.",
resolved_chat_template,
renderer_config.model_config.model,
model_config.model,
)
return resolved_chat_template
+4 -5
View File
@@ -6,7 +6,7 @@ from typing import Any, cast
from typing_extensions import assert_never
from vllm.config import RendererConfig
from vllm.config import ModelConfig
from vllm.logger import init_logger
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
from vllm.multimodal.cache import BaseMultiModalProcessorCache
@@ -45,15 +45,14 @@ logger = init_logger(__name__)
class InputPreprocessor:
def __init__(
self,
renderer_config: RendererConfig,
model_config: ModelConfig,
tokenizer: TokenizerLike | None,
mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
mm_processor_cache: BaseMultiModalProcessorCache | None = None,
) -> None:
super().__init__()
self.renderer_config = renderer_config
self.model_config = renderer_config.model_config
self.model_config = model_config
self.tokenizer = tokenizer
self.mm_registry = mm_registry
self.mm_processor_cache = mm_processor_cache
@@ -232,7 +231,7 @@ class InputPreprocessor:
def _get_mm_processor(self) -> BaseMultiModalProcessor:
if not hasattr(self, "_mm_processor"):
self._mm_processor = self.mm_registry.create_processor(
self.renderer_config,
self.model_config,
tokenizer=self.tokenizer,
cache=self.mm_processor_cache,
)
+10 -10
View File
@@ -415,7 +415,7 @@ def load_weights_using_from_2_way_softmax(
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
renderer_config = model.vllm_config.renderer_config
model_config = model.vllm_config.model_config
quant_config = model.vllm_config.quant_config
text_config = model.config.get_text_config()
@@ -447,10 +447,10 @@ def load_weights_using_from_2_way_softmax(
from vllm.tokenizers import get_tokenizer
tokenizer = get_tokenizer(
renderer_config.tokenizer,
revision=renderer_config.tokenizer_revision,
tokenizer_mode=renderer_config.tokenizer_mode,
trust_remote_code=renderer_config.trust_remote_code,
model_config.tokenizer,
revision=model_config.tokenizer_revision,
tokenizer_mode=model_config.tokenizer_mode,
trust_remote_code=model_config.trust_remote_code,
)
false_id = tokenizer.convert_tokens_to_ids(tokens[0])
@@ -473,7 +473,7 @@ def load_weights_no_post_processing(model, weights: Iterable[tuple[str, torch.Te
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
renderer_config = model.vllm_config.renderer_config
model_config = model.vllm_config.model_config
quant_config = model.vllm_config.quant_config
text_config = model.config.get_text_config()
@@ -501,10 +501,10 @@ def load_weights_no_post_processing(model, weights: Iterable[tuple[str, torch.Te
from vllm.tokenizers import get_tokenizer
tokenizer = get_tokenizer(
renderer_config.tokenizer,
revision=renderer_config.tokenizer_revision,
tokenizer_mode=renderer_config.tokenizer_mode,
trust_remote_code=renderer_config.trust_remote_code,
model_config.tokenizer,
revision=model_config.tokenizer_revision,
tokenizer_mode=model_config.tokenizer_mode,
trust_remote_code=model_config.trust_remote_code,
)
token_ids = [tokenizer.convert_tokens_to_ids(t) for t in tokens]
+2 -2
View File
@@ -377,8 +377,8 @@ class DeepseekOCRForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
self.projector_config = config.projector_config
self.text_config = config.text_config
renderer_config = vllm_config.renderer_config
tokenizer = cached_tokenizer_from_config(renderer_config)
model_config = vllm_config.model_config
tokenizer = cached_tokenizer_from_config(model_config)
self.image_token_id = tokenizer.vocab[_IMAGE_TOKEN]
self.sam_model = build_sam_vit_b()
+2 -2
View File
@@ -370,8 +370,8 @@ class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
self.projector_config = config.projector_config
self.text_config = config.text_config
renderer_config = vllm_config.renderer_config
tokenizer = cached_tokenizer_from_config(renderer_config)
model_config = vllm_config.model_config
tokenizer = cached_tokenizer_from_config(model_config)
self.image_token_id: int = tokenizer.vocab[_IMAGE_TOKEN]
self.vision = self._init_vision_module(
+3 -5
View File
@@ -18,7 +18,7 @@ from transformers.models.gemma3n import (
)
from transformers.models.siglip import SiglipImageProcessorFast
from vllm.config import RendererConfig, SpeechToTextConfig, VllmConfig
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.inputs.data import PromptType
from vllm.logger import init_logger
@@ -760,7 +760,7 @@ class Gemma3nForConditionalGeneration(
cls,
audio: np.ndarray,
stt_config: SpeechToTextConfig,
renderer_config: RendererConfig,
model_config: ModelConfig,
language: Optional[str],
task_type: Literal["transcribe", "translate"],
request_prompt: str,
@@ -798,9 +798,7 @@ class Gemma3nForConditionalGeneration(
@classmethod
def get_speech_to_text_config(
cls,
renderer_config: RendererConfig,
task_type: str,
cls, model_config: ModelConfig, task_type: str
) -> SpeechToTextConfig:
return SpeechToTextConfig(
# Let's set this to 30 as suggested in the docs for now, although
+6 -8
View File
@@ -34,7 +34,7 @@ import torch.nn.functional as F
from torch import nn
from transformers import BatchFeature, PretrainedConfig
from vllm.config import CacheConfig, RendererConfig, SpeechToTextConfig, VllmConfig
from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.inputs.data import PromptType
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
@@ -840,7 +840,7 @@ class GraniteSpeechForConditionalGeneration(
def get_generation_prompt(
cls,
audio: np.ndarray,
renderer_config: RendererConfig,
model_config: ModelConfig,
stt_config: SpeechToTextConfig,
language: str | None,
task_type: Literal["transcribe", "translate"],
@@ -861,7 +861,7 @@ class GraniteSpeechForConditionalGeneration(
else:
raise ValueError(f"Unsupported task type {task_type}")
tokenizer = cached_tokenizer_from_config(renderer_config)
tokenizer = cached_tokenizer_from_config(model_config)
chat = [dict(role="user", content=user_prompt)]
prompt = tokenizer.apply_chat_template(
chat,
@@ -882,10 +882,10 @@ class GraniteSpeechForConditionalGeneration(
cls,
audio_duration_s: float,
stt_config: SpeechToTextConfig,
renderer_config: RendererConfig,
model_config: ModelConfig,
) -> int | None:
"""Get the number of audio tokens for an audio duration in sec."""
processor = cached_processor_from_config(renderer_config)
processor = cached_processor_from_config(model_config)
hop_length = processor.audio_processor.melspec_kwargs["hop_length"]
proj_win_size = processor.audio_processor.projector_window_size
ds_rate = processor.audio_processor.projector_downsample_rate
@@ -903,9 +903,7 @@ class GraniteSpeechForConditionalGeneration(
@classmethod
def get_speech_to_text_config(
cls,
renderer_config: RendererConfig,
task_type: str,
cls, model_config: ModelConfig, task_type: str
) -> SpeechToTextConfig:
"""Get the stt config for this model."""
# Default settings are reasonable for this model and we don't currently
+7 -7
View File
@@ -6,7 +6,7 @@ import numpy as np
import torch
import torch.nn as nn
from vllm.config import RendererConfig, VllmConfig
from vllm.config import ModelConfig, VllmConfig
from vllm.logger import init_logger
from vllm.model_executor.layers.pooler import (
DispatchPooler,
@@ -29,12 +29,12 @@ logger = init_logger(__name__)
class GritLMMeanPool(nn.Module):
"""As `MeanPool`, but only includes non-instruction tokens."""
def __init__(self, renderer_config: RendererConfig):
def __init__(self, model_config: ModelConfig):
super().__init__()
self.renderer_config = renderer_config
self.model_config = model_config
tokenizer = cached_tokenizer_from_config(self.renderer_config)
tokenizer = cached_tokenizer_from_config(self.model_config)
# Collect the tokens needed for pattern matching.
# "▁<" is different from "_<". The former uses "▁" to indicate that
@@ -174,10 +174,10 @@ class GritLMMeanPool(nn.Module):
class GritLMPooler(Pooler):
def __init__(self, renderer_config: RendererConfig):
def __init__(self, model_config: ModelConfig):
super().__init__()
self.pooling = GritLMMeanPool(renderer_config)
self.pooling = GritLMMeanPool(model_config)
self.head = PoolerHead(PoolerNormalize())
def get_supported_tasks(self) -> Set[PoolingTask]:
@@ -238,6 +238,6 @@ class GritLM(LlamaForCausalLM):
self.pooler = DispatchPooler(
{
"token_embed": Pooler.for_token_embed(pooler_config),
"embed": GritLMPooler(vllm_config.renderer_config),
"embed": GritLMPooler(vllm_config.model_config),
}
)
+4 -6
View File
@@ -19,7 +19,7 @@ from torch import Tensor
from transformers.models.whisper.tokenization_whisper import LANGUAGES
from typing_extensions import Self, TypeIs
from vllm.config import RendererConfig, SpeechToTextConfig
from vllm.config import ModelConfig, SpeechToTextConfig
from vllm.inputs import TokensPrompt
from vllm.inputs.data import PromptType
from vllm.logger import init_logger
@@ -887,7 +887,7 @@ class SupportsTranscription(Protocol):
cls,
audio: np.ndarray,
stt_config: SpeechToTextConfig,
renderer_config: RendererConfig,
model_config: ModelConfig,
language: str | None,
task_type: Literal["transcribe", "translate"],
request_prompt: str,
@@ -930,9 +930,7 @@ class SupportsTranscription(Protocol):
@classmethod
def get_speech_to_text_config(
cls,
renderer_config: RendererConfig,
task_type: Literal["transcribe", "translate"],
cls, model_config: ModelConfig, task_type: Literal["transcribe", "translate"]
) -> SpeechToTextConfig:
"""Get the speech to text config for the ASR model."""
...
@@ -942,7 +940,7 @@ class SupportsTranscription(Protocol):
cls,
audio_duration_s: float,
stt_config: SpeechToTextConfig,
renderer_config: RendererConfig,
model_config: ModelConfig,
) -> int | None:
"""
Map from audio duration to number of audio tokens produced by the ASR
+1 -1
View File
@@ -182,7 +182,7 @@ class InternS1ProcessingInfo(BaseProcessingInfo):
def get_hf_processor(self, **kwargs: object) -> InternVLProcessor:
hf_processor = self.ctx.get_hf_processor(InternVLProcessor, **kwargs)
hf_processor.video_processor = cached_video_processor_from_config(
self.ctx.renderer_config,
self.ctx.model_config,
processor_cls=InternVLVideoProcessor,
size=hf_processor.image_processor.size,
**kwargs,
@@ -1169,17 +1169,16 @@ class NemotronH_Nano_VL_V2(
self.mlp1 = self.mlp1.to(self.language_model.config.dtype)
self.config = config
self.model_config = vllm_config.model_config
# Pre-tokenize special tokens for video processing
# to avoid repeated tokenization
self._tokenizer = cached_tokenizer_from_config(vllm_config.renderer_config)
self._img_start_token_ids = self._tokenizer.encode(
tokenizer = cached_tokenizer_from_config(vllm_config.model_config)
self._img_start_token_ids = tokenizer.encode(
IMG_START, add_special_tokens=False
)
self._img_end_token_ids = self._tokenizer.encode(
IMG_END, add_special_tokens=False
)
self._img_context_token_ids = self._tokenizer.encode(
self._img_end_token_ids = tokenizer.encode(IMG_END, add_special_tokens=False)
self._img_context_token_ids = tokenizer.encode(
IMG_CONTEXT, add_special_tokens=False
)
@@ -1365,7 +1364,7 @@ class NemotronH_Nano_VL_V2(
input_embeds for the LLM.
"""
device = video_embeddings.device
tokenizer = self._tokenizer
tokenizer = cached_tokenizer_from_config(self.model_config)
# Generate video replacement token IDs using get_video_repl
# This tokenizes each frame separator independently, then uses pre-tokenized
+1 -1
View File
@@ -347,7 +347,7 @@ class NemotronVLProcessingInfo(BaseInternVLProcessingInfo):
def get_image_processor(self, **kwargs: object):
return cached_image_processor_from_config(
self.ctx.renderer_config,
self.ctx.model_config,
**kwargs,
)
+1 -1
View File
@@ -193,7 +193,7 @@ class PixtralProcessorAdapter:
class PixtralProcessingInfo(BaseProcessingInfo):
def get_tokenizer(self) -> MistralTokenizer:
tokenizer = cached_tokenizer_from_config(self.ctx.renderer_config)
tokenizer = cached_tokenizer_from_config(self.ctx.model_config)
if not isinstance(tokenizer, MistralTokenizer):
raise ValueError("This model requires `--tokenizer-mode mistral`")
+10 -12
View File
@@ -20,7 +20,7 @@ from mistral_common.tokens.tokenizers.audio import Audio, AudioEncoder
from transformers import BatchFeature, TensorType, WhisperConfig
from transformers.tokenization_utils_base import TextInput
from vllm.config import RendererConfig, SpeechToTextConfig, VllmConfig
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.inputs.data import PromptType
from vllm.logger import init_logger
@@ -176,7 +176,7 @@ class VoxtralProcessorAdapter:
class VoxtralProcessingInfo(BaseProcessingInfo):
def get_tokenizer(self) -> MistralTokenizer:
tokenizer = cached_tokenizer_from_config(self.ctx.renderer_config)
tokenizer = cached_tokenizer_from_config(self.ctx.model_config)
if not isinstance(tokenizer, MistralTokenizer):
raise ValueError("This model requires `--tokenizer-mode mistral`")
@@ -339,7 +339,7 @@ class VoxtralForConditionalGeneration(
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
self.tokenizer = cached_tokenizer_from_config(vllm_config.renderer_config)
self.tokenizer = cached_tokenizer_from_config(vllm_config.model_config)
# update quant config to so that ignored module and target module names
# match the vLLM model names
@@ -450,11 +450,9 @@ class VoxtralForConditionalGeneration(
@classmethod
def get_speech_to_text_config(
cls,
renderer_config: RendererConfig,
task_type: str,
cls, model_config: ModelConfig, task_type: str
) -> SpeechToTextConfig:
tokenizer = cached_tokenizer_from_config(renderer_config)
tokenizer = cached_tokenizer_from_config(model_config)
audio_config = tokenizer.instruct.audio_encoder.audio_config
max_audio_clip_s = audio_config.chunk_length_s
sample_rate = audio_config.sampling_rate
@@ -470,17 +468,17 @@ class VoxtralForConditionalGeneration(
def get_generation_prompt(
cls,
audio: np.ndarray,
renderer_config: RendererConfig, # not needed here
model_config: ModelConfig,
stt_config: SpeechToTextConfig,
language: str | None,
task_type: Literal["transcribe", "translate"],
request_prompt: str,
to_language: str | None,
) -> PromptType:
tokenizer = cached_tokenizer_from_config(renderer_config)
tokenizer = cached_tokenizer_from_config(model_config)
audio = Audio(audio, int(stt_config.sample_rate), format="wav") # lossless
req = TranscriptionRequest(
model=renderer_config.model_config.model,
model=model_config.model,
audio=RawAudio.from_audio(audio),
language=language,
)
@@ -496,14 +494,14 @@ class VoxtralForConditionalGeneration(
cls,
audio_duration_s: float,
stt_config: SpeechToTextConfig,
renderer_config: RendererConfig,
model_config: ModelConfig,
) -> int | None:
"""
Map from audio duration to number of audio tokens produced by the ASR
model, without running a forward pass.
This is used for estimating the amount of processing for this audio.
"""
tokenizer = cached_tokenizer_from_config(renderer_config)
tokenizer = cached_tokenizer_from_config(model_config)
adapter = VoxtralProcessorAdapter(tokenizer)
return adapter.get_num_audio_tokens(
int(audio_duration_s * stt_config.sample_rate)
+6 -8
View File
@@ -19,7 +19,7 @@ from transformers.models.whisper.modeling_whisper import sinusoids
from vllm.attention.backends.abstract import AttentionType
from vllm.attention.layer import Attention, MultiHeadAttention
from vllm.attention.layers.cross_attention import CrossAttention
from vllm.config import CacheConfig, RendererConfig, SpeechToTextConfig, VllmConfig
from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.inputs.data import PromptType
@@ -811,7 +811,7 @@ class WhisperForConditionalGeneration(
def get_generation_prompt(
cls,
audio: np.ndarray,
renderer_config: RendererConfig, # not needed here
model_config: ModelConfig, # not needed here
stt_config: SpeechToTextConfig,
language: str | None,
task_type: Literal["transcribe", "translate"],
@@ -847,11 +847,9 @@ class WhisperForConditionalGeneration(
@classmethod
def get_speech_to_text_config(
cls,
renderer_config: RendererConfig,
task_type: str,
cls, model_config: ModelConfig, task_type: str
) -> SpeechToTextConfig:
processor = cached_processor_from_config(renderer_config)
processor = cached_processor_from_config(model_config)
return SpeechToTextConfig(
max_audio_clip_s=processor.feature_extractor.chunk_length,
@@ -863,9 +861,9 @@ class WhisperForConditionalGeneration(
cls,
audio_duration_s: float,
stt_config: SpeechToTextConfig,
renderer_config: RendererConfig,
model_config: ModelConfig,
) -> int | None:
processor = cached_processor_from_config(renderer_config)
processor = cached_processor_from_config(model_config)
hop_length = processor.feature_extractor.hop_length
assert hop_length is not None
# NOTE(NickLucche) user can't pass encoder
+12 -10
View File
@@ -31,7 +31,7 @@ from .inputs import (
)
if TYPE_CHECKING:
from vllm.config import ModelConfig, RendererConfig, VllmConfig
from vllm.config import ModelConfig, VllmConfig
from .processing import ResolvedPromptUpdate
from .registry import MultiModalRegistry
@@ -561,13 +561,13 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):
def _enable_processor_cache(
renderer_config: "RendererConfig",
model_config: "ModelConfig",
mm_registry: "MultiModalRegistry",
) -> bool:
if not mm_registry.supports_multimodal_inputs(renderer_config):
if not mm_registry.supports_multimodal_inputs(model_config):
return False
mm_config = renderer_config.model_config.get_multimodal_config()
mm_config = model_config.get_multimodal_config()
return mm_config.mm_processor_cache_gb > 0
@@ -599,7 +599,7 @@ def processor_cache_from_config(
"""Return a `BaseMultiModalProcessorCache`, if enabled."""
model_config = vllm_config.model_config
if not _enable_processor_cache(vllm_config.renderer_config, mm_registry):
if not _enable_processor_cache(model_config, mm_registry):
return None
if not _enable_ipc_cache(vllm_config):
@@ -611,14 +611,14 @@ def processor_cache_from_config(
def processor_only_cache_from_config(
renderer_config: "RendererConfig",
model_config: "ModelConfig",
mm_registry: "MultiModalRegistry",
):
"""Return a `MultiModalProcessorOnlyCache`, if enabled."""
if not _enable_processor_cache(renderer_config, mm_registry):
if not _enable_processor_cache(model_config, mm_registry):
return None
return MultiModalProcessorOnlyCache(renderer_config.model_config)
return MultiModalProcessorOnlyCache(model_config)
class BaseMultiModalReceiverCache(
@@ -787,7 +787,7 @@ def engine_receiver_cache_from_config(
"""
model_config = vllm_config.model_config
if not _enable_processor_cache(vllm_config.renderer_config, mm_registry):
if not _enable_processor_cache(model_config, mm_registry):
return None
if not _enable_ipc_cache(vllm_config):
@@ -809,7 +809,9 @@ def worker_receiver_cache_from_config(
Return a `BaseMultiModalReceiverCache` only when IPC caching is enabled and
mm_processor_cache_type=="shm".
"""
if not _enable_processor_cache(vllm_config.renderer_config, mm_registry):
model_config = vllm_config.model_config
if not _enable_processor_cache(model_config, mm_registry):
return None
if not _enable_ipc_cache(vllm_config):
+5 -23
View File
@@ -23,7 +23,7 @@ import torch
from typing_extensions import TypeVar, assert_never
from vllm.logger import init_logger
from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
from vllm.tokenizers import TokenizerLike
from vllm.transformers_utils.processor import cached_processor_from_config
from vllm.utils.collection_utils import flatten_2d_lists, full_groupby
from vllm.utils.func_utils import get_allowed_kwarg_only_overrides
@@ -53,7 +53,7 @@ if TYPE_CHECKING:
from transformers.feature_extraction_utils import BatchFeature
from transformers.processing_utils import ProcessorMixin
from vllm.config import ModelConfig, RendererConfig
from vllm.config import ModelConfig
from .cache import BaseMultiModalProcessorCache
from .profiling import BaseDummyInputsBuilder
@@ -63,7 +63,6 @@ else:
ProcessorMixin = object
ModelConfig = object
RendererConfig = object
BaseMultiModalProcessorCache = object
@@ -946,29 +945,12 @@ class InputProcessingContext:
modify the inputs.
"""
renderer_config: RendererConfig
"""The configuration of the renderer."""
model_config: ModelConfig
"""The configuration of the model."""
tokenizer: TokenizerLike | None
"""The tokenizer used to tokenize the inputs."""
@classmethod
def from_config(
cls,
renderer_config: RendererConfig,
*,
tokenizer: TokenizerLike | None = None,
):
if tokenizer is None and not renderer_config.skip_tokenizer_init:
tokenizer = cached_tokenizer_from_config(renderer_config)
return cls(renderer_config, tokenizer)
@property
def model_config(self) -> ModelConfig:
"""The configuration of the model."""
return self.renderer_config.model_config
def get_tokenizer(self) -> TokenizerLike:
if self.tokenizer is None:
raise ValueError(
@@ -1065,7 +1047,7 @@ class InputProcessingContext:
typ = ProcessorMixin
return cached_processor_from_config(
self.renderer_config,
self.model_config,
processor_cls=typ,
tokenizer=self.tokenizer,
**kwargs,
+34 -30
View File
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Generic, Protocol, TypeVar, cast
from vllm.config.multimodal import BaseDummyOptions
from vllm.logger import init_logger
from vllm.tokenizers import TokenizerLike
from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
from .cache import BaseMultiModalProcessorCache
from .processing import (
@@ -22,7 +22,7 @@ from .profiling import (
)
if TYPE_CHECKING:
from vllm.config import ModelConfig, RendererConfig
from vllm.config import ModelConfig
from vllm.model_executor.models.interfaces import SupportsMultiModal
logger = init_logger(__name__)
@@ -114,18 +114,17 @@ class MultiModalRegistry:
return mm_options if len(mm_options) > 0 else None
def supports_multimodal_inputs(self, renderer_config: "RendererConfig") -> bool:
def supports_multimodal_inputs(self, model_config: "ModelConfig") -> bool:
"""
Checks if the model supports multimodal inputs.
Returns True if the model is multimodal with any non-zero supported
modalities, otherwise returns False, effectively running in
text-only mode.
"""
model_config = renderer_config.model_config
if not model_config.is_multimodal_model:
return False
info = self._create_processing_info(renderer_config, tokenizer=None)
info = self._create_processing_info(model_config, tokenizer=None)
supported_modalities = info.get_supported_mm_limits()
mm_config = model_config.get_multimodal_config()
@@ -145,7 +144,7 @@ class MultiModalRegistry:
def get_max_tokens_per_item_by_modality(
self,
renderer_config: "RendererConfig",
model_config: "ModelConfig",
*,
cache: BaseMultiModalProcessorCache | None = None,
profiler_limits: Mapping[str, int] | None = None,
@@ -154,11 +153,10 @@ class MultiModalRegistry:
Get the maximum number of tokens per data item from each modality based
on underlying model configuration.
"""
model_config = renderer_config.model_config
if not model_config.is_multimodal_model:
return {}
processor = self.create_processor(renderer_config, cache=cache)
processor = self.create_processor(model_config, cache=cache)
profiler: MultiModalProfiler = MultiModalProfiler(processor)
seq_len = model_config.max_model_len
@@ -173,7 +171,7 @@ class MultiModalRegistry:
def get_mm_limits_per_prompt(
self,
renderer_config: "RendererConfig",
model_config: "ModelConfig",
*,
cache: BaseMultiModalProcessorCache | None = None,
) -> Mapping[str, int]:
@@ -181,11 +179,10 @@ class MultiModalRegistry:
Get the maximum number of multi-modal input instances for each modality
that are allowed per prompt for a model class.
"""
model_config = renderer_config.model_config
if not model_config.is_multimodal_model:
return {}
processor = self.create_processor(renderer_config, cache=cache)
processor = self.create_processor(model_config, cache=cache)
profiler: MultiModalProfiler = MultiModalProfiler(processor)
return profiler.get_mm_limits()
@@ -231,21 +228,30 @@ class MultiModalRegistry:
assert hasattr(model_cls, "_processor_factory")
return cast("SupportsMultiModal", model_cls)
def _create_processing_ctx(
self,
model_config: "ModelConfig",
tokenizer: TokenizerLike | None = None,
) -> InputProcessingContext:
if tokenizer is None and not model_config.skip_tokenizer_init:
tokenizer = cached_tokenizer_from_config(model_config)
return InputProcessingContext(model_config, tokenizer)
def _create_processing_info(
self,
renderer_config: "RendererConfig",
model_config: "ModelConfig",
*,
tokenizer: TokenizerLike | None = None,
) -> BaseProcessingInfo:
model_cls = self._get_model_cls(renderer_config.model_config)
model_cls = self._get_model_cls(model_config)
factories = model_cls._processor_factory
ctx = InputProcessingContext.from_config(renderer_config, tokenizer=tokenizer)
ctx = self._create_processing_ctx(model_config, tokenizer)
return factories.info(ctx)
def create_processor(
self,
renderer_config: "RendererConfig",
model_config: "ModelConfig",
*,
tokenizer: TokenizerLike | None = None,
cache: BaseMultiModalProcessorCache | None = None,
@@ -253,19 +259,19 @@ class MultiModalRegistry:
"""
Create a multi-modal processor for a specific model and tokenizer.
"""
model_config = renderer_config.model_config
if not model_config.is_multimodal_model:
raise ValueError(f"{model_config.model} is not a multimodal model")
model_cls = self._get_model_cls(model_config)
factories = model_cls._processor_factory
ctx = InputProcessingContext.from_config(renderer_config, tokenizer=tokenizer)
ctx = self._create_processing_ctx(model_config, tokenizer)
return factories.build_processor(ctx, cache=cache)
def get_decoder_dummy_data(
self,
renderer_config: "RendererConfig",
model_config: "ModelConfig",
seq_len: int,
mm_counts: Mapping[str, int] | None = None,
*,
@@ -274,15 +280,15 @@ class MultiModalRegistry:
"""
Create dummy data for profiling the memory usage of a model.
The model is identified by `renderer_config`.
The model is identified by `model_config`.
"""
processor = self.create_processor(renderer_config, cache=cache)
processor = self.create_processor(model_config, cache=cache)
profiler: MultiModalProfiler = MultiModalProfiler(processor)
# Extract configurable options from multimodal config.
# Only include modalities that use advanced option types so legacy
# count-only behavior remains unchanged.
mm_options = self._extract_mm_options(renderer_config.model_config)
mm_options = self._extract_mm_options(model_config)
dummy_data = profiler.get_decoder_dummy_data(seq_len, mm_counts, mm_options)
@@ -298,7 +304,7 @@ class MultiModalRegistry:
def get_encoder_dummy_data(
self,
renderer_config: "RendererConfig",
model_config: "ModelConfig",
seq_len: int,
mm_counts: Mapping[str, int] | None = None,
*,
@@ -307,15 +313,15 @@ class MultiModalRegistry:
"""
Create dummy data for profiling the memory usage of a model.
The model is identified by `renderer_config`.
The model is identified by `model_config`.
"""
processor = self.create_processor(renderer_config, cache=cache)
processor = self.create_processor(model_config, cache=cache)
profiler: MultiModalProfiler = MultiModalProfiler(processor)
# Extract configurable options from multimodal config.
# Only include modalities that use advanced option types so legacy
# count-only behavior remains unchanged.
mm_options = self._extract_mm_options(renderer_config.model_config)
mm_options = self._extract_mm_options(model_config)
dummy_data = profiler.get_encoder_dummy_data(seq_len, mm_counts, mm_options)
@@ -330,15 +336,13 @@ class MultiModalRegistry:
return dummy_data
def get_encdec_max_encoder_len(self, renderer_config: "RendererConfig") -> int:
def get_encdec_max_encoder_len(self, model_config: "ModelConfig") -> int:
"""
Get the maximum length of the encoder input for encoder-decoder models.
"""
model_config = renderer_config.model_config
if not model_config.is_encoder_decoder:
return 0
max_tokens = self.get_max_tokens_per_item_by_modality(renderer_config)
max_tokens = self.get_max_tokens_per_item_by_modality(model_config)
if not max_tokens:
# TODO - this function assumes encoder-decoder models are
# multimodal. This will need to change when adding support for more
+12 -12
View File
@@ -24,7 +24,7 @@ from vllm.utils.import_utils import resolve_obj_by_qualname
from .protocol import TokenizerLike
if TYPE_CHECKING:
from vllm.config import RendererConfig
from vllm.config import ModelConfig
logger = init_logger(__name__)
@@ -205,18 +205,18 @@ def get_tokenizer(
cached_get_tokenizer = lru_cache(get_tokenizer)
def cached_tokenizer_from_config(renderer_config: "RendererConfig", **kwargs):
def cached_tokenizer_from_config(model_config: "ModelConfig", **kwargs):
return cached_get_tokenizer(
renderer_config.tokenizer,
tokenizer_mode=renderer_config.tokenizer_mode,
revision=renderer_config.tokenizer_revision,
trust_remote_code=renderer_config.trust_remote_code,
model_config.tokenizer,
tokenizer_mode=model_config.tokenizer_mode,
revision=model_config.tokenizer_revision,
trust_remote_code=model_config.trust_remote_code,
**kwargs,
)
def init_tokenizer_from_config(renderer_config: "RendererConfig"):
runner_type = renderer_config.model_config.runner_type
def init_tokenizer_from_config(model_config: "ModelConfig"):
runner_type = model_config.runner_type
if runner_type == "generate" or runner_type == "draft":
truncation_side = "left"
elif runner_type == "pooling":
@@ -225,9 +225,9 @@ def init_tokenizer_from_config(renderer_config: "RendererConfig"):
assert_never(runner_type)
return get_tokenizer(
renderer_config.tokenizer,
tokenizer_mode=renderer_config.tokenizer_mode,
trust_remote_code=renderer_config.trust_remote_code,
revision=renderer_config.tokenizer_revision,
model_config.tokenizer,
tokenizer_mode=model_config.tokenizer_mode,
trust_remote_code=model_config.trust_remote_code,
revision=model_config.tokenizer_revision,
truncation_side=truncation_side,
)
+11 -17
View File
@@ -23,7 +23,7 @@ from vllm.transformers_utils.utils import convert_model_repo_to_path
from vllm.utils.func_utils import get_allowed_kwarg_only_overrides
if TYPE_CHECKING:
from vllm.config import ModelConfig, RendererConfig
from vllm.config import ModelConfig
_P = TypeVar("_P", bound=ProcessorMixin, default=ProcessorMixin)
_V = TypeVar("_V", bound=BaseVideoProcessor, default=BaseVideoProcessor)
@@ -233,18 +233,17 @@ def cached_get_processor_without_dynamic_kwargs(
def cached_processor_from_config(
renderer_config: "RendererConfig",
model_config: "ModelConfig",
processor_cls: type[_P] | tuple[type[_P], ...] = ProcessorMixin,
**kwargs: Any,
) -> _P:
model_config = renderer_config.model_config
if is_gguf(model_config.model):
assert not is_gguf(renderer_config.tokenizer), (
assert not is_gguf(model_config.tokenizer), (
"For multimodal GGUF models, the original tokenizer "
"should be used to correctly load processor."
)
model = renderer_config.tokenizer
revision = renderer_config.tokenizer_revision
model = model_config.tokenizer
revision = model_config.tokenizer_revision
else:
model = model_config.model
revision = model_config.revision
@@ -298,11 +297,9 @@ cached_get_feature_extractor = lru_cache(get_feature_extractor)
def cached_feature_extractor_from_config(
renderer_config: "RendererConfig",
model_config: "ModelConfig",
**kwargs: Any,
):
model_config = renderer_config.model_config
return cached_get_feature_extractor(
model_config.model,
revision=model_config.revision,
@@ -351,17 +348,16 @@ cached_get_image_processor = lru_cache(get_image_processor)
def cached_image_processor_from_config(
renderer_config: "RendererConfig",
model_config: "ModelConfig",
**kwargs: Any,
):
model_config = renderer_config.model_config
if is_gguf(model_config.model):
assert not is_gguf(renderer_config.tokenizer), (
assert not is_gguf(model_config.tokenizer), (
"For multimodal GGUF models, the original tokenizer "
"should be used to correctly load image processor."
)
model = renderer_config.tokenizer
revision = renderer_config.tokenizer_revision
model = model_config.tokenizer
revision = model_config.tokenizer_revision
else:
model = model_config.model
revision = model_config.revision
@@ -415,12 +411,10 @@ cached_get_video_processor = lru_cache(get_video_processor)
def cached_video_processor_from_config(
renderer_config: "RendererConfig",
model_config: "ModelConfig",
processor_cls: type[_V] | None = None,
**kwargs: Any,
):
model_config = renderer_config.model_config
return cached_get_video_processor(
model_config.model,
revision=model_config.revision,
+4 -4
View File
@@ -10,7 +10,7 @@ from vllm.multimodal import MultiModalRegistry
from vllm.v1.request import Request
if TYPE_CHECKING:
from vllm.config import RendererConfig, SchedulerConfig
from vllm.config import ModelConfig, SchedulerConfig
logger = init_logger(__name__)
@@ -250,7 +250,7 @@ class EncoderCacheManager:
def compute_encoder_budget(
renderer_config: "RendererConfig",
model_config: "ModelConfig",
scheduler_config: "SchedulerConfig",
mm_registry: MultiModalRegistry,
) -> tuple[int, int]:
@@ -263,9 +263,9 @@ def compute_encoder_budget(
- Space budget for encoder cache size, measured in number of tokens
from the input sequence.
"""
if mm_registry.supports_multimodal_inputs(renderer_config):
if mm_registry.supports_multimodal_inputs(model_config):
max_tokens_by_modality = mm_registry.get_max_tokens_per_item_by_modality(
renderer_config
model_config
)
return compute_mm_encoder_budget(
+1 -1
View File
@@ -164,7 +164,7 @@ class Scheduler(SchedulerInterface):
# This can be changed when we make encoder cache for embedding caching
# across requests.
encoder_compute_budget, encoder_cache_size = compute_encoder_budget(
renderer_config=vllm_config.renderer_config,
model_config=vllm_config.model_config,
scheduler_config=vllm_config.scheduler_config,
mm_registry=mm_registry,
)
+3 -4
View File
@@ -91,7 +91,6 @@ class AsyncLLM(EngineClient):
# Ensure we can serialize custom transformer configs
maybe_register_config_serialize_by_value()
self.renderer_config = vllm_config.renderer_config
self.model_config = vllm_config.model_config
self.vllm_config = vllm_config
self.observability_config = vllm_config.observability_config
@@ -109,15 +108,15 @@ class AsyncLLM(EngineClient):
"enabling logging without default stat loggers."
)
if self.renderer_config.skip_tokenizer_init:
if self.model_config.skip_tokenizer_init:
tokenizer = None
else:
tokenizer = init_tokenizer_from_config(self.renderer_config)
tokenizer = init_tokenizer_from_config(self.model_config)
self.input_processor = InputProcessor(self.vllm_config, tokenizer)
self.io_processor = get_io_processor(
self.vllm_config,
self.renderer_config.io_processor_plugin,
self.model_config.io_processor_plugin,
)
# OutputProcessor (converts EngineCoreOutputs --> RequestOutput).
+3 -4
View File
@@ -43,7 +43,6 @@ class InputProcessor:
mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
) -> None:
self.vllm_config = vllm_config
self.renderer_config = vllm_config.renderer_config
self.model_config = vllm_config.model_config
self.cache_config = vllm_config.cache_config
self.lora_config = vllm_config.lora_config
@@ -55,7 +54,7 @@ class InputProcessor:
self.mm_processor_cache = processor_cache_from_config(vllm_config, mm_registry)
self.input_preprocessor = InputPreprocessor(
self.renderer_config,
self.model_config,
tokenizer,
mm_registry,
mm_processor_cache=self.mm_processor_cache,
@@ -253,7 +252,7 @@ class InputProcessor:
if not params.structured_outputs or not self.structured_outputs_config:
return
if self.renderer_config.skip_tokenizer_init and params.structured_outputs:
if self.model_config.skip_tokenizer_init and params.structured_outputs:
raise ValueError(
"Structured outputs requires a tokenizer so it can't be used with 'skip_tokenizer_init'" # noqa: E501
)
@@ -583,7 +582,7 @@ class InputProcessor:
if prompt_type == "encoder" and model_config.is_multimodal_model:
mm_registry = self.input_preprocessor.mm_registry
mm_processor = mm_registry.create_processor(
self.renderer_config,
model_config,
tokenizer=tokenizer,
)
assert isinstance(mm_processor, EncDecMultiModalProcessor)
+3 -4
View File
@@ -60,7 +60,6 @@ class LLMEngine:
) -> None:
self.vllm_config = vllm_config
self.observability_config = vllm_config.observability_config
self.renderer_config = vllm_config.renderer_config
self.model_config = vllm_config.model_config
self.cache_config = vllm_config.cache_config
@@ -84,15 +83,15 @@ class LLMEngine:
self.dp_group = None
self.should_execute_dummy_batch = False
if self.renderer_config.skip_tokenizer_init:
if self.model_config.skip_tokenizer_init:
tokenizer = None
else:
tokenizer = init_tokenizer_from_config(self.renderer_config)
tokenizer = init_tokenizer_from_config(self.model_config)
self.input_processor = InputProcessor(self.vllm_config, tokenizer)
self.io_processor = get_io_processor(
self.vllm_config,
self.renderer_config.io_processor_plugin,
self.model_config.io_processor_plugin,
)
# OutputProcessor (convert EngineCoreOutputs --> RequestOutput).
+1 -1
View File
@@ -85,7 +85,7 @@ class EagleProposer:
# Multi-modal data support
self.mm_registry = MULTIMODAL_REGISTRY
self.supports_mm_inputs = self.mm_registry.supports_multimodal_inputs(
vllm_config.renderer_config
vllm_config.model_config
)
self.attn_metadata_builder: AttentionMetadataBuilder | None = None
+12 -6
View File
@@ -63,7 +63,7 @@ class StructuredOutputManager:
max_workers = max(1, min(multiprocessing.cpu_count() // 2, 8))
self.executor_for_fillmask = ThreadPoolExecutor(max_workers=max_workers)
if not vllm_config.renderer_config.skip_tokenizer_init:
if not self.vllm_config.model_config.skip_tokenizer_init:
# The default max_workers if not specified is the number of
# CPUs * 5, which is way too high since these tasks are CPU-bound,
# not I/O bound. We also know we would never dominate CPU usage
@@ -71,15 +71,21 @@ class StructuredOutputManager:
# of CPUs.
max_workers = max(1, (multiprocessing.cpu_count() + 1) // 2)
self.executor = ThreadPoolExecutor(max_workers=max_workers)
self.tokenizer = init_tokenizer_from_config(vllm_config.renderer_config)
reasoning_parser = vllm_config.structured_outputs_config.reasoning_parser
self.tokenizer = init_tokenizer_from_config(
model_config=self.vllm_config.model_config
)
reasoning_parser = (
self.vllm_config.structured_outputs_config.reasoning_parser
)
reasoning_parser_plugin = (
vllm_config.structured_outputs_config.reasoning_parser_plugin
self.vllm_config.structured_outputs_config.reasoning_parser_plugin
)
if reasoning_parser_plugin and len(reasoning_parser_plugin) > 3:
ReasoningParserManager.import_reasoning_parser(reasoning_parser_plugin)
reasoning_parser = vllm_config.structured_outputs_config.reasoning_parser
reasoning_parser = (
self.vllm_config.structured_outputs_config.reasoning_parser
)
if reasoning_parser:
reasoner_cls = ReasoningParserManager.get_reasoning_parser(
reasoning_parser
@@ -87,7 +93,7 @@ class StructuredOutputManager:
self.reasoner = reasoner_cls(tokenizer=self.tokenizer)
self.enable_in_reasoning = (
vllm_config.structured_outputs_config.enable_in_reasoning
self.vllm_config.structured_outputs_config.enable_in_reasoning
)
def grammar_init(self, request: Request) -> None:
+3 -4
View File
@@ -271,7 +271,6 @@ class GPUModelRunner(
device: torch.device,
):
self.vllm_config = vllm_config
self.renderer_config = vllm_config.renderer_config
self.model_config = vllm_config.model_config
self.cache_config = vllm_config.cache_config
self.compilation_config = vllm_config.compilation_config
@@ -336,7 +335,7 @@ class GPUModelRunner(
self.uses_mrope = model_config.uses_mrope
self.uses_xdrope_dim = model_config.uses_xdrope_dim
self.supports_mm_inputs = self.mm_registry.supports_multimodal_inputs(
self.renderer_config
model_config
)
if self.model_config.is_encoder_decoder:
@@ -559,7 +558,7 @@ class GPUModelRunner(
self.mm_budget = (
MultiModalBudget(
self.renderer_config,
self.model_config,
self.scheduler_config,
self.mm_registry,
)
@@ -3874,7 +3873,7 @@ class GPUModelRunner(
assert self.mm_budget is not None
dummy_decoder_data = self.mm_registry.get_decoder_dummy_data(
renderer_config=self.renderer_config,
model_config=self.model_config,
seq_len=self.max_model_len,
mm_counts={modality: 1},
cache=self.mm_budget.cache,
+3 -4
View File
@@ -143,7 +143,6 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
original_parallel_config: ParallelConfig | None = None,
):
self.vllm_config = vllm_config
self.renderer_config = vllm_config.renderer_config
self.model_config = vllm_config.model_config
self.cache_config = vllm_config.cache_config
self.lora_config = vllm_config.lora_config
@@ -223,7 +222,7 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
self.mm_registry = MULTIMODAL_REGISTRY
self.uses_mrope = model_config.uses_mrope
self.supports_mm_inputs = self.mm_registry.supports_multimodal_inputs(
self.renderer_config
model_config
)
# TODO: Support M-RoPE (e.g, Qwen2-VL)
assert not self.uses_mrope, "TPU does not support M-RoPE yet."
@@ -354,7 +353,7 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
self.mm_budget = (
MultiModalBudget(
self.renderer_config,
self.model_config,
self.scheduler_config,
self.mm_registry,
)
@@ -2039,7 +2038,7 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
assert self.mm_budget is not None
dummy_decoder_data = self.mm_registry.get_decoder_dummy_data(
renderer_config=self.renderer_config,
model_config=self.model_config,
seq_len=self.max_model_len,
mm_counts={modality: 1},
cache=self.mm_budget.cache,
+7 -12
View File
@@ -7,7 +7,7 @@ import torch
from vllm.attention.backends.abstract import AttentionBackend
from vllm.attention.layer import Attention
from vllm.config import RendererConfig, SchedulerConfig, VllmConfig
from vllm.config import ModelConfig, SchedulerConfig, VllmConfig
from vllm.model_executor.models.interfaces import MultiModalEmbeddings
from vllm.model_executor.models.utils import extract_layer_index
from vllm.multimodal.cache import processor_only_cache_from_config
@@ -23,29 +23,24 @@ class MultiModalBudget:
def __init__(
self,
renderer_config: RendererConfig,
model_config: ModelConfig,
scheduler_config: SchedulerConfig,
mm_registry: MultiModalRegistry,
) -> None:
super().__init__()
self.renderer_config = renderer_config
self.model_config = renderer_config.model_config
self.model_config = model_config
self.scheduler_config = scheduler_config
self.mm_registry = mm_registry
self.cache = cache = processor_only_cache_from_config(
renderer_config, mm_registry
)
self.cache = cache = processor_only_cache_from_config(model_config, mm_registry)
self.max_model_len = self.model_config.max_model_len
self.max_model_len = model_config.max_model_len
self.max_num_reqs = scheduler_config.max_num_seqs
self.mm_limits = mm_registry.get_mm_limits_per_prompt(
renderer_config, cache=cache
)
self.mm_limits = mm_registry.get_mm_limits_per_prompt(model_config, cache=cache)
max_tokens_by_modality = mm_registry.get_max_tokens_per_item_by_modality(
renderer_config,
model_config,
cache=cache,
profiler_limits=self.mm_limits,
)