vLLM/vllm/engine/arg_utils.py

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

import argparse
import copy
import dataclasses
import functools
import json
import os
import sys
import warnings
from collections.abc import Callable
from dataclasses import MISSING, asdict, dataclass, fields, is_dataclass
from types import UnionType
from typing import (
    TYPE_CHECKING,
    Annotated,
    Any,
    Literal,
    TypeAlias,
    TypeVar,
    Union,
    cast,
    get_args,
    get_origin,
    is_typeddict,
)

import huggingface_hub
import regex as re
import torch
from pydantic import TypeAdapter, ValidationError
from pydantic.fields import FieldInfo
from typing_extensions import TypeIs

import vllm.envs as envs
from vllm.config import (
    AttentionConfig,
    AuxOutputConfig,
    CacheConfig,
    CompilationConfig,
    ConfigType,
    DeviceConfig,
    DiffusionConfig,
    ECTransferConfig,
    EncoderCacheManagerConfig,
    EngramConfig,
    EPLBConfig,
    FaultToleranceConfig,
    KernelConfig,
    KVEventsConfig,
    KVTransferConfig,
    LoadConfig,
    LoggingConfig,
    LoRAConfig,
    MambaConfig,
    ModelConfig,
    MultiModalConfig,
    ObservabilityConfig,
    OffloadConfig,
    ParallelConfig,
    PoolerConfig,
    PrefetchOffloadConfig,
    ProfilerConfig,
    ReasoningConfig,
    SchedulerConfig,
    SpeculativeConfig,
    StructuredOutputsConfig,
    UVAOffloadConfig,
    VllmConfig,
    WeightTransferConfig,
    get_attr_docs,
)
from vllm.config.cache import (
    CacheDType,
    KVOffloadingBackend,
    MambaCacheMode,
    MambaDType,
    PrefixCachingHashAlgo,
)
from vllm.config.device import Device
from vllm.config.kernel import (
    PASSTHROUGH_ALL2ALL_BACKEND,
    IrOpPriorityConfig,
    LinearBackend,
    MoEBackend,
    SparseIndexerTopkBackend,
)
from vllm.config.load import SafetensorsLoadStrategy
from vllm.config.logging import LogLevel
from vllm.config.lora import MaxLoRARanks
from vllm.config.mamba import MambaBackendEnum, MambaSSUAlgorithm
from vllm.config.model import (
    ConvertOption,
    HfOverrides,
    LogprobsMode,
    ModelDType,
    RunnerOption,
    TokenizerMode,
)
from vllm.config.multimodal import (
    MMCacheType,
    MMEncoderTPMode,
    MMHasherAlgorithm,
    MMProcessorDevice,
    MMTensorIPC,
)
from vllm.config.observability import DetailedTraceModules
from vllm.config.parallel import (
    All2AllBackend,
    DataParallelBackend,
    DCPCommBackend,
    DistributedExecutorBackend,
    ExpertPlacementStrategy,
)
from vllm.config.scheduler import SchedulerPolicy
from vllm.config.utils import get_field
from vllm.config.vllm import OptimizationLevel, PerformanceMode
from vllm.config.watermarking import WatermarkConfig
from vllm.logger import configure_logging_if_needed, init_logger, suppress_logging
from vllm.platforms import CpuArchEnum, current_platform
from vllm.plugins import load_general_plugins
from vllm.ray.lazy_utils import is_in_ray_actor, is_ray_initialized
from vllm.transformers_utils.config import maybe_override_with_speculators
from vllm.transformers_utils.repo_utils import get_model_path
from vllm.transformers_utils.utils import is_cloud_storage
from vllm.utils.argparse_utils import (
    FlexibleArgumentParser,
    human_readable_int,
    human_readable_int_or_auto,
)
from vllm.utils.mem_constants import GiB_bytes
from vllm.utils.network_utils import get_ip
from vllm.utils.torch_utils import resolve_kv_cache_dtype_string
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.sample.logits_processor import LogitsProcessor
from vllm.version import __version__ as VLLM_VERSION

if TYPE_CHECKING:
    from vllm.config.quantization import QuantizationConfigArgs
    from vllm.model_executor.layers.quantization import QuantizationMethods
    from vllm.model_executor.model_loader import LoadFormats
    from vllm.usage.usage_lib import UsageContext
    from vllm.v1.executor import Executor
else:
    Executor = Any
    QuantizationMethods = str
    LoadFormats = str
    UsageContext = Any


logger = init_logger(__name__)

# object is used to allow for special typing forms
T = TypeVar("T")
TypeHint: TypeAlias = type[Any] | object
TypeHintT: TypeAlias = type[T] | object


def parse_type(return_type: Callable[[str], T]) -> Callable[[str], T]:
    def _parse_type(val: str) -> T:
        try:
            return return_type(val)
        except ValueError as e:
            raise argparse.ArgumentTypeError(
                f"Value {val} cannot be converted to {return_type}."
            ) from e

    return _parse_type


def optional_type(return_type: Callable[[str], T]) -> Callable[[str], T | None]:
    def _optional_type(val: str) -> T | None:
        if val == "" or val == "None":
            return None
        return parse_type(return_type)(val)

    return _optional_type


def union_dict_and_str(val: str) -> str | dict[str, str] | None:
    if not re.match(r"(?s)^\s*{.*}\s*$", val):
        return str(val)
    return optional_type(json.loads)(val)


def parse_reasoning_parser(val: str) -> str:
    """Map `none` (any case) to the unset default, as `vllm-rs serve` does."""
    return "" if val.lower() == "none" else val


def is_type(type_hint: TypeHint, type: TypeHintT) -> TypeIs[TypeHintT]:
    """Check if the type hint is a specific type."""
    return type_hint is type or get_origin(type_hint) is type


def is_dict_subclass(type_hint: TypeHint) -> bool:
    """Check if the type hint is a subclass of `dict`.

    `TypedDict`s are excluded because they do not support class checks.
    """
    return (
        isinstance(type_hint, type)
        and not is_typeddict(type_hint)
        and issubclass(type_hint, dict)
    )


def contains_type(type_hints: set[TypeHint], type: TypeHintT) -> bool:
    """Check if the type hints contain a specific type."""
    return any(is_type(type_hint, type) for type_hint in type_hints)


def get_type(type_hints: set[TypeHint], type: TypeHintT) -> TypeHintT:
    """Get the specific type from the type hints."""
    return next((th for th in type_hints if is_type(th, type)), None)


def literal_to_kwargs(type_hints: set[TypeHint]) -> dict[str, Any]:
    """Get the `type` and `choices` from a `Literal` type hint in `type_hints`.

    If `type_hints` also contains `str`, we use `metavar` instead of `choices`.
    """
    type_hint = get_type(type_hints, Literal)
    options = get_args(type_hint)
    option_type = type(options[0])
    if not all(isinstance(option, option_type) for option in options):
        raise ValueError(
            "All options must be of the same type. "
            f"Got {options} with types {[type(c) for c in options]}"
        )
    kwarg = "metavar" if contains_type(type_hints, str) else "choices"
    return {"type": option_type, kwarg: sorted(options)}


def collection_to_kwargs(type_hints: set[TypeHint], type: TypeHint) -> dict[str, Any]:
    type_hint = get_type(type_hints, type)
    types = get_args(type_hint)
    elem_type = types[0]

    # Handle Ellipsis
    assert all(t is elem_type for t in types if t is not Ellipsis), (
        f"All non-Ellipsis elements must be of the same type. Got {types}."
    )

    # Handle Union types
    if get_origin(elem_type) in {Union, UnionType}:
        # Union for Union[X, Y] and UnionType for X | Y
        assert str in get_args(elem_type), (
            "If element can have multiple types, one must be 'str' "
            f"(i.e. 'list[int | str]'). Got {elem_type}."
        )
        elem_type = str

    return {
        "type": elem_type,
        "nargs": "+" if type is not tuple or Ellipsis in types else len(types),
    }


def is_not_builtin(type_hint: TypeHint) -> bool:
    """Check if the class is not a built-in type."""
    return type_hint.__module__ != "builtins"


def get_type_hints(type_hint: TypeHint) -> set[TypeHint]:
    """Extract type hints from Annotated or Union type hints."""
    type_hints: set[TypeHint] = set()
    origin = get_origin(type_hint)
    args = get_args(type_hint)

    if origin is Annotated:
        type_hints.update(get_type_hints(args[0]))
    elif origin in {Union, UnionType}:
        # Union for Union[X, Y] and UnionType for X | Y
        for arg in args:
            type_hints.update(get_type_hints(arg))
    else:
        type_hints.add(type_hint)

    return type_hints


NEEDS_HELP = (
    any("--help" in arg for arg in sys.argv)  # vllm SUBCOMMAND --help
    or "mkdocs" in sys.modules  # mkdocs SUBCOMMAND
)


def _maybe_add_docs_url(cls: Any) -> str:
    """Generate API docs URL for a vllm config class."""
    import vllm.config

    name = cls.__name__
    if getattr(vllm.config, name, None) is not cls:
        return ""
    version = f"v{VLLM_VERSION}" if "dev" not in VLLM_VERSION else "latest"
    return f"\n\nAPI docs: https://docs.vllm.ai/en/{version}/api/vllm/config/#vllm.config.{name}"


def _expand_json_human_readable_numbers(val: str) -> str:
    """Expand human-readable number suffixes in a JSON string.

    Based on :func:`human_readable_int` so that the ``k/m/g/t`` (decimal) and
    ``K/M/G/T`` (binary) conventions work out the box.
    Also works inside JSON config arguments such
    as ``--kv-transfer-config '{"cpu_bytes_to_use": 80m}'``.

    Only bare (unquoted) tokens are replaced so that JSON string values
    like ``"model_name"`` are never modified.
    """
    # Split on quoted strings so we only touch non-string regions.
    parts = re.split(r'("(?:[^"\\]|\\.)*")', val)
    for i in range(0, len(parts), 2):  # even indices = outside strings
        parts[i] = re.sub(
            r"\b\d+(?:\.\d+)?[kKmMgGtT]\b",
            lambda m: str(human_readable_int(m.group())),
            parts[i],
        )
    return "".join(parts)


@functools.lru_cache(maxsize=30)
def _compute_kwargs(cls: ConfigType) -> dict[str, dict[str, Any]]:
    # Save time only getting attr docs if we're generating help text
    cls_docs = get_attr_docs(cls) if NEEDS_HELP else {}
    kwargs = {}
    for field in fields(cls):
        # Get the set of possible types for the field
        type_hints: set[TypeHint] = get_type_hints(field.type)
        # Subclasses of dict (e.g. MultiModalDummyOptions) are CLI dicts
        type_hints = {dict if is_dict_subclass(th) else th for th in type_hints}

        # If the field is a dataclass, we can use the model_validate_json
        generator = (th for th in type_hints if is_dataclass(th))
        dataclass_cls = next(generator, None)

        # Get the default value of the field
        if field.default is not MISSING:
            default = field.default
            # Handle pydantic.Field defaults
            if isinstance(default, FieldInfo):
                if default.default_factory is None:
                    default = default.default
                else:
                    # VllmConfig's Fields have default_factory set to config classes.
                    # These could emit logs on init, which would be confusing.
                    with suppress_logging():
                        default = default.default_factory()  # type: ignore[call-arg]
        elif field.default_factory is not MISSING:
            default = field.default_factory()

        # Get the help text for the field
        name = field.name
        help = cls_docs.get(name, "").strip()
        # Escape % for argparse
        help = help.replace("%", "%%")

        # Initialise the kwargs dictionary for the field
        kwargs[name] = {"default": default, "help": help}

        # Set other kwargs based on the type hints
        json_tip = (
            "Should either be a valid JSON string or JSON keys passed individually."
        )
        if dataclass_cls is not None:

            def parse_dataclass(val: str, cls=dataclass_cls) -> Any:
                try:
                    val = _expand_json_human_readable_numbers(val)
                    return TypeAdapter(cls).validate_json(val)
                except ValidationError as e:
                    raise argparse.ArgumentTypeError(repr(e)) from e

            kwargs[name]["type"] = parse_dataclass
            kwargs[name]["help"] += _maybe_add_docs_url(dataclass_cls)
            kwargs[name]["help"] += f"\n\n{json_tip}"
        elif type_hints == {bool, str, type(None)}:
            # Optional-valued flag: bare flag -> True, value -> str.
            kwargs[name]["type"] = str
            kwargs[name]["nargs"] = "?"
            kwargs[name]["const"] = True
        elif contains_type(type_hints, bool):
            # Creates --no-<name> and --<name> flags
            kwargs[name]["action"] = argparse.BooleanOptionalAction
        elif contains_type(type_hints, Literal):
            kwargs[name].update(literal_to_kwargs(type_hints))
        elif contains_type(type_hints, tuple):
            kwargs[name].update(collection_to_kwargs(type_hints, tuple))
        elif contains_type(type_hints, list):
            kwargs[name].update(collection_to_kwargs(type_hints, list))
        elif contains_type(type_hints, set):
            kwargs[name].update(collection_to_kwargs(type_hints, set))
        elif contains_type(type_hints, int):
            # Arguments that accept human-readable integer strings (e.g., 1K, 2M, 1G)
            human_readable_int_args = {
                "max_num_batched_tokens",
                "max_num_scheduled_tokens",
                "kv_cache_memory_bytes",
                "safetensors_prefetch_block_size",
                "max_num_queued_tokens",
            }
            if name == "max_model_len":
                kwargs[name]["type"] = human_readable_int_or_auto
                kwargs[name]["help"] += f"\n\n{human_readable_int_or_auto.__doc__}"
            elif name in human_readable_int_args:
                kwargs[name]["type"] = human_readable_int
                kwargs[name]["help"] += f"\n\n{human_readable_int.__doc__}"
            else:
                kwargs[name]["type"] = int
        elif contains_type(type_hints, float):
            kwargs[name]["type"] = float
        elif contains_type(type_hints, dict) and (
            contains_type(type_hints, str)
            or any(is_not_builtin(th) for th in type_hints)
        ):
            kwargs[name]["type"] = union_dict_and_str
        elif contains_type(type_hints, dict):
            kwargs[name]["type"] = parse_type(json.loads)
            kwargs[name]["help"] += f"\n\n{json_tip}"
        elif contains_type(type_hints, str) or any(
            is_not_builtin(th) for th in type_hints
        ):
            kwargs[name]["type"] = str
        else:
            raise ValueError(f"Unsupported type {type_hints} for argument {name}.")

        # If the type hint was a sequence of literals, use the helper function
        # to update the type and choices
        if get_origin(kwargs[name].get("type")) is Literal:
            kwargs[name].update(literal_to_kwargs({kwargs[name]["type"]}))

        # If None is in type_hints, make the argument optional.
        # But not if it's a bool, argparse will handle this better.
        if type(None) in type_hints and not contains_type(type_hints, bool):
            kwargs[name]["type"] = optional_type(kwargs[name]["type"])
            if kwargs[name].get("choices"):
                kwargs[name]["choices"].append(None)
    return kwargs


def get_kwargs(cls: ConfigType) -> dict[str, dict[str, Any]]:
    """Return argparse kwargs for the given Config dataclass.

    If `--help` or `mkdocs` are not present in the command line command, the
    attribute documentation will not be included in the help output.

    The heavy computation is cached via functools.lru_cache, and a deep copy
    is returned so callers can mutate the dictionary without affecting the
    cached version.
    """
    return copy.deepcopy(_compute_kwargs(cls))


_LOG_CONFIG_FILE_DEPRECATION_MESSAGE = (
    "--log-config-file is deprecated and will be removed in v0.33.0. "
    "Use --logging-config.pylogging_config_file instead."
)


_MAMBA_CACHE_MODE_DEPRECATION_MESSAGE = (
    "--mamba-cache-mode is deprecated and will be removed in v0.33.0. "
    "It has no effect: Mamba prefix caching follows --enable-prefix-caching."
)


_DISABLE_CASCADE_ATTN_DEPRECATION_MESSAGE = (
    "--disable-cascade-attn is deprecated and has no effect since cascade "
    "attention has been removed. It will be removed in v0.33.0."
)


class DeprecatedLogConfigFileAction(argparse.Action):
    """Warn when the legacy ``--log-config-file`` option is used."""

    def __call__(self, parser, namespace, values, option_string=None):
        warnings.warn(_LOG_CONFIG_FILE_DEPRECATION_MESSAGE, stacklevel=1)
        setattr(namespace, self.dest, values)


@dataclass
class EngineArgs:
    """Arguments for vLLM engine."""

    model: str = ModelConfig.model
    # Public compatibility argument. Canonical runtime state lives in
    # AuxOutputConfig.
    enable_return_routed_experts: bool = False
    return_sampling_mask: bool = ModelConfig.return_sampling_mask
    model_weights: str = ModelConfig.model_weights
    served_model_name: str | list[str] | None = ModelConfig.served_model_name
    tokenizer: str | None = ModelConfig.tokenizer
    hf_config_path: str | None = ModelConfig.hf_config_path
    runner: RunnerOption = ModelConfig.runner
    convert: ConvertOption = ModelConfig.convert
    skip_tokenizer_init: bool = ModelConfig.skip_tokenizer_init
    enable_prompt_embeds: bool = ModelConfig.enable_prompt_embeds
    tokenizer_mode: TokenizerMode | str = ModelConfig.tokenizer_mode
    trust_remote_code: bool = ModelConfig.trust_remote_code
    allowed_local_media_path: str = ModelConfig.allowed_local_media_path
    allowed_media_domains: list[str] | None = ModelConfig.allowed_media_domains
    download_dir: str | None = LoadConfig.download_dir
    safetensors_load_strategy: SafetensorsLoadStrategy | None = (
        LoadConfig.safetensors_load_strategy
    )
    safetensors_prefetch_num_threads: int = LoadConfig.safetensors_prefetch_num_threads
    safetensors_prefetch_block_size: int = LoadConfig.safetensors_prefetch_block_size
    load_format: str | LoadFormats = LoadConfig.load_format
    config_format: str = ModelConfig.config_format
    dtype: ModelDType = ModelConfig.dtype
    kv_cache_dtype: CacheDType = CacheConfig.cache_dtype
    seed: int = ModelConfig.seed
    max_model_len: int = ModelConfig.max_model_len
    cudagraph_capture_sizes: list[int] | None = (
        CompilationConfig.cudagraph_capture_sizes
    )
    max_cudagraph_capture_size: int | None = get_field(
        CompilationConfig, "max_cudagraph_capture_size"
    )
    ir_op_priority: IrOpPriorityConfig = get_field(KernelConfig, "ir_op_priority")
    # Note: Specifying a custom executor backend by passing a class
    # is intended for expert use only. The API may change without
    # notice.
    distributed_executor_backend: (
        str | DistributedExecutorBackend | type[Executor] | None
    ) = ParallelConfig.distributed_executor_backend
    # number of P/D disaggregation (or other disaggregation) workers
    pipeline_parallel_size: int = ParallelConfig.pipeline_parallel_size
    master_addr: str = ParallelConfig.master_addr
    master_port: int = ParallelConfig.master_port
    nnodes: int = ParallelConfig.nnodes
    node_rank: int = ParallelConfig.node_rank
    distributed_timeout_seconds: int | None = ParallelConfig.distributed_timeout_seconds
    cpu_distributed_timeout_seconds: int | None = (
        ParallelConfig.cpu_distributed_timeout_seconds
    )
    numa_bind: bool = ParallelConfig.numa_bind
    numa_bind_nodes: list[int] | None = ParallelConfig.numa_bind_nodes
    numa_bind_cpus: list[str] | None = ParallelConfig.numa_bind_cpus
    device_ids: list[int | str] | None = None
    tensor_parallel_size: int = ParallelConfig.tensor_parallel_size
    prefill_context_parallel_size: int = ParallelConfig.prefill_context_parallel_size
    decode_context_parallel_size: int = ParallelConfig.decode_context_parallel_size
    dcp_comm_backend: DCPCommBackend | None = ParallelConfig.dcp_comm_backend
    dcp_q_replicate: bool | None = ParallelConfig.dcp_q_replicate
    dcp_kv_cache_interleave_size: int = ParallelConfig.dcp_kv_cache_interleave_size
    cp_kv_cache_interleave_size: int | None = None
    data_parallel_size: int = ParallelConfig.data_parallel_size
    data_parallel_rank: int | None = None
    data_parallel_start_rank: int | None = None
    data_parallel_size_local: int | None = None
    data_parallel_address: str | None = None
    data_parallel_rpc_port: int | None = None
    data_parallel_hybrid_lb: bool = False
    data_parallel_external_lb: bool = False
    data_parallel_multi_port_external_lb: bool = False
    data_parallel_backend: DataParallelBackend = ParallelConfig.data_parallel_backend
    enable_hc_sp: bool = ParallelConfig.enable_hc_sp
    enable_expert_parallel: bool = ParallelConfig.enable_expert_parallel
    enable_batch_sharded_sampling: bool | None = (
        ParallelConfig.enable_batch_sharded_sampling
    )
    enable_ep_weight_filter: bool = ParallelConfig.enable_ep_weight_filter
    moe_backend: MoEBackend = KernelConfig.moe_backend
    linear_backend: LinearBackend = KernelConfig.linear_backend
    sparse_indexer_topk_backend: SparseIndexerTopkBackend = (
        KernelConfig.sparse_indexer_topk_backend
    )
    all2all_backend: All2AllBackend = ParallelConfig.all2all_backend
    enable_elastic_ep: bool = ParallelConfig.enable_elastic_ep
    elastic_ep_max_dp_size: int = ParallelConfig.elastic_ep_max_dp_size
    enable_dbo: bool = ParallelConfig.enable_dbo
    ubatch_size: int = ParallelConfig.ubatch_size
    dbo_decode_token_threshold: int = ParallelConfig.dbo_decode_token_threshold
    dp_sync_interval: int = ParallelConfig.dp_sync_interval
    dbo_prefill_token_threshold: int = ParallelConfig.dbo_prefill_token_threshold
    disable_nccl_for_dp_synchronization: bool | None = (
        ParallelConfig.disable_nccl_for_dp_synchronization
    )
    eplb_config: EPLBConfig = get_field(ParallelConfig, "eplb_config")
    enable_eplb: bool = ParallelConfig.enable_eplb
    expert_placement_strategy: ExpertPlacementStrategy = (
        ParallelConfig.expert_placement_strategy
    )
    _api_process_count: int = ParallelConfig._api_process_count
    _api_process_rank: int = ParallelConfig._api_process_rank
    max_parallel_loading_workers: int | None = (
        ParallelConfig.max_parallel_loading_workers
    )
    block_size: int | None = None
    enable_prefix_caching: bool | None = None
    prefix_caching_hash_algo: PrefixCachingHashAlgo = (
        CacheConfig.prefix_caching_hash_algo
    )
    prefix_cache_retention_interval: int | None = get_field(
        CacheConfig, "prefix_cache_retention_interval"
    )
    disable_sliding_window: bool = ModelConfig.disable_sliding_window
    disable_cascade_attn: bool | None = None
    offload_backend: str = OffloadConfig.offload_backend
    cpu_offload_gb: float = UVAOffloadConfig.cpu_offload_gb
    cpu_offload_params: set[str] = get_field(UVAOffloadConfig, "cpu_offload_params")
    offload_group_size: int = PrefetchOffloadConfig.offload_group_size
    offload_num_in_group: int = PrefetchOffloadConfig.offload_num_in_group
    offload_prefetch_step: int = PrefetchOffloadConfig.offload_prefetch_step
    offload_params: set[str] = get_field(PrefetchOffloadConfig, "offload_params")
    gpu_memory_utilization: float = CacheConfig.gpu_memory_utilization
    kv_cache_memory_bytes: int | None = CacheConfig.kv_cache_memory_bytes
    max_num_batched_tokens: int | None = None
    max_num_scheduled_tokens: int | None = None
    long_prefill_token_threshold: int = SchedulerConfig.long_prefill_token_threshold
    long_prefill_token_threshold_adaptive: bool = (
        SchedulerConfig.long_prefill_token_threshold_adaptive
    )
    max_num_seqs: int | None = None
    max_num_active_seqs: int | None = SchedulerConfig.max_num_active_seqs
    max_num_queued_reqs: int | None = None
    max_num_queued_tokens: int | None = None
    max_logprobs: int = ModelConfig.max_logprobs
    logprobs_mode: LogprobsMode = ModelConfig.logprobs_mode
    use_fp64_gumbel: bool = ModelConfig.use_fp64_gumbel
    enable_trace_replay: bool = ModelConfig.enable_trace_replay
    disable_log_stats: bool = False
    aggregate_engine_logging: bool = False
    revision: str | None = ModelConfig.revision
    code_revision: str | None = ModelConfig.code_revision
    hf_token: bool | str | None = ModelConfig.hf_token
    hf_overrides: HfOverrides = get_field(ModelConfig, "hf_overrides")
    model_class_overrides: dict[str, str] = get_field(
        ModelConfig, "model_class_overrides"
    )
    tokenizer_revision: str | None = ModelConfig.tokenizer_revision
    quantization: QuantizationMethods | str | None = ModelConfig.quantization
    quantization_config: "dict[str, Any] | QuantizationConfigArgs | None" = None
    """User-facing quantization configuration. Carries per-layer-kind
    QuantSpecs (linear, moe) and ignore patterns; see
    :class:`QuantizationConfigArgs`. Auto-populated from the matching online
    shorthand when `quantization` is one of the values in
    `ONLINE_QUANT_SHORTHAND_NAMES`."""
    allow_deprecated_quantization: bool = ModelConfig.allow_deprecated_quantization
    enforce_eager: bool = ModelConfig.enforce_eager
    disable_custom_all_reduce: bool = ParallelConfig.disable_custom_all_reduce
    enable_shm_tensor_arena: bool = ParallelConfig.enable_shm_tensor_arena
    language_model_only: bool = MultiModalConfig.language_model_only
    limit_mm_per_prompt: dict[str, int | dict[str, int]] = get_field(
        MultiModalConfig, "limit_per_prompt"
    )
    enable_mm_embeds: bool = MultiModalConfig.enable_mm_embeds
    interleave_mm_strings: bool = MultiModalConfig.interleave_mm_strings
    media_io_kwargs: dict[str, dict[str, Any]] = get_field(
        MultiModalConfig, "media_io_kwargs"
    )
    mm_processor_kwargs: dict[str, Any] | None = MultiModalConfig.mm_processor_kwargs
    mm_processor_cache_gb: float = MultiModalConfig.mm_processor_cache_gb
    mm_processor_cache_type: MMCacheType | None = (
        MultiModalConfig.mm_processor_cache_type
    )
    mm_hasher_algorithm: MMHasherAlgorithm = get_field(
        MultiModalConfig, "mm_hasher_algorithm"
    )
    mm_shm_cache_max_object_size_mb: int = (
        MultiModalConfig.mm_shm_cache_max_object_size_mb
    )
    mm_encoder_only: bool = MultiModalConfig.mm_encoder_only
    mm_encoder_tp_mode: MMEncoderTPMode = MultiModalConfig.mm_encoder_tp_mode
    mm_encoder_attn_backend: AttentionBackendEnum | str | None = (
        MultiModalConfig.mm_encoder_attn_backend
    )
    mm_encoder_attn_dtype: str | None = MultiModalConfig.mm_encoder_attn_dtype
    mm_encoder_fp8_scale_path: str | None = MultiModalConfig.mm_encoder_fp8_scale_path
    mm_encoder_fp8_scale_save_path: str | None = (
        MultiModalConfig.mm_encoder_fp8_scale_save_path
    )
    mm_encoder_fp8_scale_save_margin: float = (
        MultiModalConfig.mm_encoder_fp8_scale_save_margin
    )
    io_processor_plugin: str | None = None
    renderer_num_workers: int = 1
    skip_mm_profiling: bool = MultiModalConfig.skip_mm_profiling
    video_pruning_rate: float | None = MultiModalConfig.video_pruning_rate
    video_pruning_method: str = MultiModalConfig.video_pruning_method
    mm_tensor_ipc: MMTensorIPC = MultiModalConfig.mm_tensor_ipc
    mm_processor_device: MMProcessorDevice = "auto"
    mm_ipc_gpu_memory_gb: float = MultiModalConfig.mm_ipc_gpu_memory_gb
    mm_device_do_normalize: bool | None = MultiModalConfig.mm_device_do_normalize
    # LoRA fields
    enable_lora: bool = False
    max_loras: int = LoRAConfig.max_loras
    max_lora_rank: MaxLoRARanks = LoRAConfig.max_lora_rank
    default_mm_loras: dict[str, str] | None = LoRAConfig.default_mm_loras
    fully_sharded_loras: bool = LoRAConfig.fully_sharded_loras
    max_cpu_loras: int | None = LoRAConfig.max_cpu_loras
    lora_dtype: str | torch.dtype | None = LoRAConfig.lora_dtype
    lora_target_modules: list[str] | None = LoRAConfig.target_modules
    enable_tower_connector_lora: bool = LoRAConfig.enable_tower_connector_lora
    specialize_active_lora: bool = LoRAConfig.specialize_active_lora
    enable_mixed_moe_lora_format: bool = LoRAConfig.enable_mixed_moe_lora_format
    enable_moe_shared_loras: bool = LoRAConfig.enable_moe_shared_loras
    max_lora_cls_labels: int | None = LoRAConfig.max_lora_cls_labels

    ray_workers_use_nsight: bool = ParallelConfig.ray_workers_use_nsight
    num_gpu_blocks_override: int | None = CacheConfig.num_gpu_blocks_override
    model_loader_extra_config: dict = get_field(LoadConfig, "model_loader_extra_config")
    ignore_patterns: str | list[str] = get_field(LoadConfig, "ignore_patterns")

    enable_chunked_prefill: bool | None = None
    disable_chunked_mm_input: bool = SchedulerConfig.disable_chunked_mm_input

    scheduler_reserve_full_isl: bool = SchedulerConfig.scheduler_reserve_full_isl
    prefill_schedule_interval: int = SchedulerConfig.prefill_schedule_interval

    watermark: float = SchedulerConfig.watermark

    disable_hybrid_kv_cache_manager: bool | None = (
        SchedulerConfig.disable_hybrid_kv_cache_manager
    )

    structured_outputs_config: StructuredOutputsConfig = get_field(
        VllmConfig, "structured_outputs_config"
    )
    aux_output_config: AuxOutputConfig = get_field(VllmConfig, "aux_output_config")
    reasoning_parser: str = StructuredOutputsConfig.reasoning_parser
    reasoning_parser_plugin: str | None = None

    speculative_config: dict[str, Any] | None = None
    spec_method: str | None = None
    spec_model: str | None = None
    spec_tokens: int | None = None
    watermark_config: dict[str, Any] | None = None
    diffusion_config: dict[str, Any] | None = None

    show_hidden_metrics_for_version: str | None = (
        ObservabilityConfig.show_hidden_metrics_for_version
    )
    otlp_traces_endpoint: str | None = ObservabilityConfig.otlp_traces_endpoint
    collect_detailed_traces: list[DetailedTraceModules] | None = (
        ObservabilityConfig.collect_detailed_traces
    )
    per_request_spec_decode_metrics: Literal["none", "summary", "detailed"] = (
        ObservabilityConfig.per_request_spec_decode_metrics
    )
    kv_cache_metrics: bool = ObservabilityConfig.kv_cache_metrics
    kv_cache_metrics_sample: float = get_field(
        ObservabilityConfig, "kv_cache_metrics_sample"
    )
    custom_histogram_buckets: dict[str, list[float]] | None = (
        ObservabilityConfig.custom_histogram_buckets
    )
    cudagraph_metrics: bool = ObservabilityConfig.cudagraph_metrics
    enable_layerwise_nvtx_tracing: bool = (
        ObservabilityConfig.enable_layerwise_nvtx_tracing
    )
    enable_mfu_metrics: bool = ObservabilityConfig.enable_mfu_metrics
    enable_logging_iteration_details: bool = (
        ObservabilityConfig.enable_logging_iteration_details
    )
    jit_monitor_mode: Literal["warn", "error"] = ObservabilityConfig.jit_monitor_mode
    jit_monitor_verbose: bool = ObservabilityConfig.jit_monitor_verbose
    enable_mm_processor_stats: bool = ObservabilityConfig.enable_mm_processor_stats
    scheduling_policy: SchedulerPolicy = SchedulerConfig.policy
    scheduler_cls: str | type[object] | None = SchedulerConfig.scheduler_cls

    pooler_config: PoolerConfig | None = ModelConfig.pooler_config
    compilation_config: CompilationConfig = get_field(VllmConfig, "compilation_config")
    attention_config: AttentionConfig = get_field(VllmConfig, "attention_config")
    engram_config: EngramConfig | None = VllmConfig.engram_config
    mamba_config: MambaConfig = get_field(VllmConfig, "mamba_config")
    kernel_config: KernelConfig = get_field(VllmConfig, "kernel_config")
    enable_flashinfer_autotune: bool = get_field(
        KernelConfig, "enable_flashinfer_autotune"
    )
    monokernel: bool = get_field(KernelConfig, "monokernel")
    worker_cls: str = ParallelConfig.worker_cls
    worker_extension_cls: str = ParallelConfig.worker_extension_cls

    profiler_config: ProfilerConfig = get_field(VllmConfig, "profiler_config")

    logging_config: LoggingConfig | None = None
    log_level: LogLevel | None = None
    log_config_file: str | None = None

    kv_transfer_config: KVTransferConfig | None = None
    kv_events_config: KVEventsConfig | None = None

    ec_transfer_config: ECTransferConfig | None = None
    ec_manager_config: EncoderCacheManagerConfig = get_field(
        VllmConfig, "ec_manager_config"
    )
    reasoning_config: ReasoningConfig = get_field(VllmConfig, "reasoning_config")

    generation_config: str = ModelConfig.generation_config
    enable_sleep_mode: bool = ModelConfig.enable_sleep_mode
    sleep_mode_offload_cudagraph: bool = ModelConfig.sleep_mode_offload_cudagraph
    sleep_preserve_parameter_names: list[str] = get_field(
        ModelConfig, "sleep_preserve_parameter_names"
    )
    enable_cumem_allocator: bool = ModelConfig.enable_cumem_allocator
    enable_nccl_comm_suspend: bool = ModelConfig.enable_nccl_comm_suspend
    override_generation_config: dict[str, Any] = get_field(
        ModelConfig, "override_generation_config"
    )
    model_impl: str = ModelConfig.model_impl
    attention_backend: AttentionBackendEnum | None = AttentionConfig.backend

    kv_cache_dtype_skip_layers: list[str] = get_field(
        CacheConfig, "kv_cache_dtype_skip_layers"
    )
    mamba_cache_dtype: MambaDType = CacheConfig.mamba_cache_dtype
    mamba_ssm_cache_dtype: MambaDType = CacheConfig.mamba_ssm_cache_dtype
    mamba_block_size: int | None = get_field(CacheConfig, "mamba_block_size")
    prefix_match_unit: int | None = get_field(CacheConfig, "prefix_match_unit")
    mamba_cache_mode: MambaCacheMode | None = None
    enable_mamba_shared_prefix_checkpoint: bool = (
        CacheConfig.enable_mamba_shared_prefix_checkpoint
    )
    replayssm_buffer_len: int = CacheConfig.replayssm_buffer_len
    use_replayssm: bool = CacheConfig.use_replayssm

    mamba_backend: MambaBackendEnum = MambaBackendEnum.TRITON
    mamba_ssu_algorithm: MambaSSUAlgorithm | None = None
    enable_mamba_cache_stochastic_rounding: bool = (
        MambaConfig.enable_stochastic_rounding
    )
    mamba_cache_philox_rounds: int = MambaConfig.stochastic_rounding_philox_rounds

    additional_config: dict[str, Any] = get_field(VllmConfig, "additional_config")

    use_tqdm_on_load: bool = LoadConfig.use_tqdm_on_load
    pt_load_map_location: str | dict[str, str] = LoadConfig.pt_load_map_location

    logits_processors: list[str | type[LogitsProcessor]] | None = (
        ModelConfig.logits_processors
    )
    """Custom logitproc types"""

    async_scheduling: bool | None = SchedulerConfig.async_scheduling

    stream_interval: int = SchedulerConfig.stream_interval

    kv_sharing_fast_prefill: bool = CacheConfig.kv_sharing_fast_prefill
    swa_bounded_replay: bool = CacheConfig.swa_bounded_replay
    optimization_level: OptimizationLevel = VllmConfig.optimization_level
    performance_mode: PerformanceMode = VllmConfig.performance_mode

    fault_tolerance_config: FaultToleranceConfig = get_field(
        ParallelConfig, "fault_tolerance_config"
    )
    enable_fault_tolerance: bool = ParallelConfig.enable_fault_tolerance

    kv_offloading_size: float | None = CacheConfig.kv_offloading_size
    kv_offloading_backend: KVOffloadingBackend = CacheConfig.kv_offloading_backend
    tokens_only: bool = False

    shutdown_timeout: int = 0

    weight_transfer_config: WeightTransferConfig | None = get_field(
        VllmConfig,
        "weight_transfer_config",
    )

    fail_on_environ_validation: bool = False
    gdn_prefill_backend: (
        Literal["flashinfer", "triton", "cutedsl", "aiter_flydsl"] | None
    ) = None
    kda_prefill_backend: (
        Literal["auto", "triton", "flashkda", "flashinfer", "fused"] | None
    ) = None
    kda_decode_backend: Literal["auto", "native", "flashinfer", "triton"] | None = None

    def __post_init__(self):
        # support `EngineArgs(compilation_config={...})`
        # without having to manually construct a
        # CompilationConfig object
        if isinstance(self.compilation_config, dict):
            self.compilation_config = CompilationConfig(**self.compilation_config)
        if isinstance(self.attention_config, dict):
            self.attention_config = AttentionConfig(**self.attention_config)
        if isinstance(self.aux_output_config, dict):
            self.aux_output_config = AuxOutputConfig(**self.aux_output_config)
        if self.enable_return_routed_experts:
            self.aux_output_config.enable_return_routed_experts = True
        if isinstance(self.engram_config, dict):
            self.engram_config = EngramConfig(**self.engram_config)
        if isinstance(self.mamba_config, dict):
            self.mamba_config = MambaConfig(**self.mamba_config)
        if isinstance(self.kernel_config, dict):
            self.kernel_config = KernelConfig(**self.kernel_config)
        if isinstance(self.ec_manager_config, dict):
            self.ec_manager_config = EncoderCacheManagerConfig(**self.ec_manager_config)
        if isinstance(self.eplb_config, dict):
            self.eplb_config = EPLBConfig(**self.eplb_config)
        if isinstance(self.weight_transfer_config, dict):
            self.weight_transfer_config = WeightTransferConfig(
                **self.weight_transfer_config
            )
        if isinstance(self.fault_tolerance_config, dict):
            if not self.enable_fault_tolerance:
                logger.warning(
                    "--fault-tolerance-config was passed. Fault tolerance is being "
                    "automatically enabled."
                )
                self.enable_fault_tolerance = True
            self.fault_tolerance_config = FaultToleranceConfig(
                **self.fault_tolerance_config
            )
        if isinstance(self.ir_op_priority, dict):
            self.ir_op_priority = IrOpPriorityConfig(**self.ir_op_priority)

        from vllm.config.quantization import resolve_quantization_config

        self.quantization_config = resolve_quantization_config(
            self.quantization, self.quantization_config
        )

        # Setup plugins
        from vllm.plugins import load_general_plugins

        load_general_plugins()
        # when use hf offline,replace model and tokenizer id to local model path
        if huggingface_hub.constants.HF_HUB_OFFLINE:
            # Skip cloud storage URIs (s3://, gs://, az://) — they are not
            # HF repo IDs and will be resolved later by
            # ModelConfig.maybe_pull_model_tokenizer_for_runai().
            if not is_cloud_storage(self.model):
                model_id = self.model
                self.model = get_model_path(self.model, self.revision)
                if model_id is not self.model:
                    logger.info(
                        "HF_HUB_OFFLINE is True, replace model_id "
                        "[%s] to model_path [%s]",
                        model_id,
                        self.model,
                    )
            if self.tokenizer is not None and not is_cloud_storage(self.tokenizer):
                tokenizer_id = self.tokenizer
                self.tokenizer = get_model_path(self.tokenizer, self.tokenizer_revision)
                if tokenizer_id is not self.tokenizer:
                    logger.info(
                        "HF_HUB_OFFLINE is True, replace tokenizer_id [%s] "
                        "to tokenizer_path [%s]",
                        tokenizer_id,
                        self.tokenizer,
                    )

    @staticmethod
    def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
        """Shared CLI arguments for vLLM engine."""
        # Model arguments
        model_kwargs = get_kwargs(ModelConfig)
        aux_output_kwargs = get_kwargs(AuxOutputConfig)
        model_group = parser.add_argument_group(
            title="ModelConfig",
            description=ModelConfig.__doc__,
        )
        if not ("serve" in sys.argv[1:] and "--help" in sys.argv[1:]):
            model_group.add_argument("--model", **model_kwargs["model"])
        model_group.add_argument("--runner", **model_kwargs["runner"])
        model_group.add_argument("--convert", **model_kwargs["convert"])
        model_group.add_argument("--tokenizer", **model_kwargs["tokenizer"])
        model_group.add_argument("--tokenizer-mode", **model_kwargs["tokenizer_mode"])
        model_group.add_argument(
            "--trust-remote-code", **model_kwargs["trust_remote_code"]
        )
        model_group.add_argument("--dtype", **model_kwargs["dtype"])
        model_group.add_argument("--seed", **model_kwargs["seed"])
        model_group.add_argument("--hf-config-path", **model_kwargs["hf_config_path"])
        model_group.add_argument(
            "--allowed-local-media-path", **model_kwargs["allowed_local_media_path"]
        )
        model_group.add_argument(
            "--allowed-media-domains", **model_kwargs["allowed_media_domains"]
        )
        model_group.add_argument("--revision", **model_kwargs["revision"])
        model_group.add_argument("--code-revision", **model_kwargs["code_revision"])
        model_group.add_argument(
            "--tokenizer-revision", **model_kwargs["tokenizer_revision"]
        )
        model_group.add_argument("--max-model-len", **model_kwargs["max_model_len"])
        model_group.add_argument("--quantization", "-q", **model_kwargs["quantization"])
        model_group.add_argument(
            "--quantization-config", **model_kwargs["quantization_config"]
        )
        model_group.add_argument(
            "--allow-deprecated-quantization",
            **model_kwargs["allow_deprecated_quantization"],
        )
        model_group.add_argument("--enforce-eager", **model_kwargs["enforce_eager"])
        model_group.add_argument(
            "--enable-return-routed-experts",
            **aux_output_kwargs["enable_return_routed_experts"],
        )
        model_group.add_argument(
            "--return-sampling-mask",
            **model_kwargs["return_sampling_mask"],
        )
        model_group.add_argument("--max-logprobs", **model_kwargs["max_logprobs"])
        model_group.add_argument("--logprobs-mode", **model_kwargs["logprobs_mode"])
        model_group.add_argument("--use-fp64-gumbel", **model_kwargs["use_fp64_gumbel"])
        model_group.add_argument(
            "--enable-trace-replay", **model_kwargs["enable_trace_replay"]
        )
        model_group.add_argument(
            "--disable-sliding-window", **model_kwargs["disable_sliding_window"]
        )
        model_group.add_argument(
            "--disable-cascade-attn",
            action=argparse.BooleanOptionalAction,
            default=None,
            help=_DISABLE_CASCADE_ATTN_DEPRECATION_MESSAGE,
        )
        model_group.add_argument(
            "--skip-tokenizer-init", **model_kwargs["skip_tokenizer_init"]
        )
        model_group.add_argument(
            "--enable-prompt-embeds", **model_kwargs["enable_prompt_embeds"]
        )
        model_group.add_argument(
            "--served-model-name", **model_kwargs["served_model_name"]
        )
        model_group.add_argument("--config-format", **model_kwargs["config_format"])
        model_group.add_argument("--hf-token", **model_kwargs["hf_token"])
        model_group.add_argument("--hf-overrides", **model_kwargs["hf_overrides"])
        model_group.add_argument(
            "--model-class-overrides", **model_kwargs["model_class_overrides"]
        )
        model_group.add_argument("--pooler-config", **model_kwargs["pooler_config"])
        model_group.add_argument(
            "--generation-config", **model_kwargs["generation_config"]
        )
        model_group.add_argument(
            "--override-generation-config", **model_kwargs["override_generation_config"]
        )
        model_group.add_argument(
            "--enable-sleep-mode", **model_kwargs["enable_sleep_mode"]
        )
        model_group.add_argument(
            "--sleep-mode-offload-cudagraph",
            **model_kwargs["sleep_mode_offload_cudagraph"],
        )
        model_group.add_argument(
            "--sleep-preserve-parameter-names",
            **model_kwargs["sleep_preserve_parameter_names"],
        )
        model_group.add_argument(
            "--enable-cumem-allocator", **model_kwargs["enable_cumem_allocator"]
        )
        model_group.add_argument(
            "--enable-nccl-comm-suspend",
            **model_kwargs["enable_nccl_comm_suspend"],
        )
        model_group.add_argument("--model-impl", **model_kwargs["model_impl"])
        model_group.add_argument(
            "--logits-processors", **model_kwargs["logits_processors"]
        )
        model_group.add_argument(
            "--io-processor-plugin", **model_kwargs["io_processor_plugin"]
        )
        model_group.add_argument(
            "--renderer-num-workers",
            **model_kwargs["renderer_num_workers"],
        )

        # Model loading arguments
        load_kwargs = get_kwargs(LoadConfig)
        load_group = parser.add_argument_group(
            title="LoadConfig",
            description=LoadConfig.__doc__,
        )
        load_group.add_argument("--load-format", **load_kwargs["load_format"])
        load_group.add_argument("--download-dir", **load_kwargs["download_dir"])
        load_group.add_argument(
            "--safetensors-load-strategy", **load_kwargs["safetensors_load_strategy"]
        )
        load_group.add_argument(
            "--safetensors-prefetch-num-threads",
            **load_kwargs["safetensors_prefetch_num_threads"],
        )
        load_group.add_argument(
            "--safetensors-prefetch-block-size",
            **load_kwargs["safetensors_prefetch_block_size"],
        )
        load_group.add_argument(
            "--model-loader-extra-config", **load_kwargs["model_loader_extra_config"]
        )
        load_group.add_argument("--ignore-patterns", **load_kwargs["ignore_patterns"])
        load_group.add_argument("--use-tqdm-on-load", **load_kwargs["use_tqdm_on_load"])
        load_group.add_argument(
            "--pt-load-map-location", **load_kwargs["pt_load_map_location"]
        )

        # Attention arguments
        attention_kwargs = get_kwargs(AttentionConfig)
        attention_group = parser.add_argument_group(
            title="AttentionConfig",
            description=AttentionConfig.__doc__,
        )
        attention_group.add_argument(
            "--attention-backend", **attention_kwargs["backend"]
        )

        # Mamba arguments
        mamba_kwargs = get_kwargs(MambaConfig)
        mamba_group = parser.add_argument_group(
            title="MambaConfig",
            description=MambaConfig.__doc__,
        )
        mamba_group.add_argument("--mamba-backend", **mamba_kwargs["backend"])
        mamba_group.add_argument(
            "--mamba-ssu-algorithm", **mamba_kwargs["ssu_algorithm"]
        )
        mamba_group.add_argument(
            "--enable-mamba-cache-stochastic-rounding",
            **mamba_kwargs["enable_stochastic_rounding"],
        )
        mamba_group.add_argument(
            "--mamba-cache-philox-rounds",
            **mamba_kwargs["stochastic_rounding_philox_rounds"],
        )

        # Structured outputs arguments
        structured_outputs_kwargs = get_kwargs(StructuredOutputsConfig)
        structured_outputs_kwargs["reasoning_parser"]["type"] = parse_reasoning_parser
        structured_outputs_kwargs["reasoning_parser"]["help"] += (
            "\n\n`none` disables reasoning parsing, the same as leaving it unset."
        )
        structured_outputs_group = parser.add_argument_group(
            title="StructuredOutputsConfig",
            description=StructuredOutputsConfig.__doc__,
        )
        structured_outputs_group.add_argument(
            "--reasoning-parser",
            # Choices need to be validated after parsing to include plugins
            **structured_outputs_kwargs["reasoning_parser"],
        )
        structured_outputs_group.add_argument(
            "--reasoning-parser-plugin",
            **structured_outputs_kwargs["reasoning_parser_plugin"],
        )

        # Parallel arguments
        parallel_kwargs = get_kwargs(ParallelConfig)
        # Bound from --moe-backend in KernelConfig.set_platform_defaults().
        parallel_kwargs["all2all_backend"]["choices"].remove(
            PASSTHROUGH_ALL2ALL_BACKEND
        )
        parallel_group = parser.add_argument_group(
            title="ParallelConfig",
            description=ParallelConfig.__doc__,
        )
        parallel_group.add_argument(
            "--distributed-executor-backend",
            **parallel_kwargs["distributed_executor_backend"],
        )
        parallel_group.add_argument(
            "--pipeline-parallel-size",
            "-pp",
            **parallel_kwargs["pipeline_parallel_size"],
        )
        parallel_group.add_argument("--master-addr", **parallel_kwargs["master_addr"])
        parallel_group.add_argument("--master-port", **parallel_kwargs["master_port"])
        parallel_group.add_argument("--nnodes", "-n", **parallel_kwargs["nnodes"])
        parallel_group.add_argument("--node-rank", "-r", **parallel_kwargs["node_rank"])
        parallel_group.add_argument(
            "--distributed-timeout-seconds",
            **parallel_kwargs["distributed_timeout_seconds"],
        )
        parallel_group.add_argument(
            "--cpu-distributed-timeout-seconds",
            **parallel_kwargs["cpu_distributed_timeout_seconds"],
        )
        parallel_group.add_argument("--numa-bind", **parallel_kwargs["numa_bind"])
        parallel_group.add_argument(
            "--numa-bind-nodes", **parallel_kwargs["numa_bind_nodes"]
        )
        parallel_group.add_argument(
            "--numa-bind-cpus", **parallel_kwargs["numa_bind_cpus"]
        )
        parallel_group.add_argument(
            "--device-ids",
            type=lambda s: [
                int(device_id) if device_id.isdigit() else device_id
                for device_id in (part.strip() for part in s.split(","))
            ],
            default=None,
            help="Comma-separated physical GPU device IDs or UUIDs to use "
            '(e.g. --device-ids "2,3,5,7"). Avoids setting '
            "CUDA_VISIBLE_DEVICES, preserving full GPU topology "
            "visibility for GPU-NIC affinity and DeepGEMM. "
            "Note: has no effect with Ray executors; use Ray "
            "placement groups for GPU selection instead.",
        )
        parallel_group.add_argument(
            "--tensor-parallel-size", "-tp", **parallel_kwargs["tensor_parallel_size"]
        )
        parallel_group.add_argument(
            "--decode-context-parallel-size",
            "-dcp",
            **parallel_kwargs["decode_context_parallel_size"],
        )
        parallel_group.add_argument(
            "--dcp-comm-backend",
            **parallel_kwargs["dcp_comm_backend"],
        )
        parallel_group.add_argument(
            "--dcp-q-replicate",
            **parallel_kwargs["dcp_q_replicate"],
        )
        parallel_group.add_argument(
            "--dcp-kv-cache-interleave-size",
            **parallel_kwargs["dcp_kv_cache_interleave_size"],
        )
        parallel_group.add_argument(
            "--cp-kv-cache-interleave-size",
            **{
                **parallel_kwargs["cp_kv_cache_interleave_size"],
                "default": None,
            },
        )
        parallel_group.add_argument(
            "--prefill-context-parallel-size",
            "-pcp",
            **parallel_kwargs["prefill_context_parallel_size"],
        )
        parallel_group.add_argument(
            "--data-parallel-size", "-dp", **parallel_kwargs["data_parallel_size"]
        )
        parallel_group.add_argument(
            "--data-parallel-rank",
            "-dpn",
            type=int,
            help="Data parallel rank of this instance. "
            "When set, enables external load balancer mode for MoE "
            "data-parallel deployments. Unsupported for non-MoE models; "
            "launch independent vLLM instances instead.",
        )
        parallel_group.add_argument(
            "--data-parallel-start-rank",
            "-dpr",
            type=int,
            help="Starting data parallel rank for secondary nodes.",
        )
        parallel_group.add_argument(
            "--data-parallel-size-local",
            "-dpl",
            type=int,
            help="Number of data parallel replicas to run on this node.",
        )
        parallel_group.add_argument(
            "--data-parallel-address",
            "-dpa",
            type=str,
            help="Address of data parallel cluster head-node.",
        )
        parallel_group.add_argument(
            "--data-parallel-rpc-port",
            "-dpp",
            type=int,
            help="Fixed port for data parallel RPC communication. All nodes "
            "must use the same port.",
        )
        parallel_group.add_argument(
            "--data-parallel-backend",
            "-dpb",
            type=str,
            default="mp",
            help='Backend for data parallel, either "mp" or "ray".',
        )
        parallel_group.add_argument(
            "--data-parallel-hybrid-lb",
            "-dph",
            **parallel_kwargs["data_parallel_hybrid_lb"],
        )
        parallel_group.add_argument(
            "--data-parallel-external-lb",
            "-dpe",
            **parallel_kwargs["data_parallel_external_lb"],
        )
        parallel_group.add_argument(
            "--data-parallel-multi-port-external-lb",
            "-dpm",
            action="store_true",
            default=False,
            help="Run a node-local supervisor that launches one external-LB API "
            "server per local data parallel rank and exposes aggregated health on "
            "a supervisor port.",
        )
        parallel_group.add_argument("--enable-hc-sp", **parallel_kwargs["enable_hc_sp"])
        parallel_group.add_argument(
            "--enable-expert-parallel",
            "-ep",
            **parallel_kwargs["enable_expert_parallel"],
        )
        parallel_group.add_argument(
            "--enable-batch-sharded-sampling",
            **parallel_kwargs["enable_batch_sharded_sampling"],
        )
        parallel_group.add_argument(
            "--enable-ep-weight-filter",
            **parallel_kwargs["enable_ep_weight_filter"],
        )
        parallel_group.add_argument(
            "--all2all-backend", **parallel_kwargs["all2all_backend"]
        )
        parallel_group.add_argument("--enable-dbo", **parallel_kwargs["enable_dbo"])
        parallel_group.add_argument(
            "--enable-shm-tensor-arena",
            **parallel_kwargs["enable_shm_tensor_arena"],
        )
        parallel_group.add_argument(
            "--ubatch-size",
            **parallel_kwargs["ubatch_size"],
        )
        parallel_group.add_argument(
            "--enable-elastic-ep", **parallel_kwargs["enable_elastic_ep"]
        )
        parallel_group.add_argument(
            "--elastic-ep-max-dp-size",
            **parallel_kwargs["elastic_ep_max_dp_size"],
        )
        parallel_group.add_argument(
            "--dbo-decode-token-threshold",
            **parallel_kwargs["dbo_decode_token_threshold"],
        )
        parallel_group.add_argument(
            "--dp-sync-interval",
            **parallel_kwargs["dp_sync_interval"],
        )
        parallel_group.add_argument(
            "--dbo-prefill-token-threshold",
            **parallel_kwargs["dbo_prefill_token_threshold"],
        )
        parallel_group.add_argument(
            "--disable-nccl-for-dp-synchronization",
            **parallel_kwargs["disable_nccl_for_dp_synchronization"],
        )
        parallel_group.add_argument("--enable-eplb", **parallel_kwargs["enable_eplb"])
        parallel_group.add_argument("--eplb-config", **parallel_kwargs["eplb_config"])
        parallel_group.add_argument(
            "--expert-placement-strategy",
            **parallel_kwargs["expert_placement_strategy"],
        )

        parallel_group.add_argument(
            "--max-parallel-loading-workers",
            **parallel_kwargs["max_parallel_loading_workers"],
        )
        parallel_group.add_argument(
            "--ray-workers-use-nsight", **parallel_kwargs["ray_workers_use_nsight"]
        )
        parallel_group.add_argument(
            "--disable-custom-all-reduce",
            **parallel_kwargs["disable_custom_all_reduce"],
        )
        parallel_group.add_argument("--worker-cls", **parallel_kwargs["worker_cls"])
        parallel_group.add_argument(
            "--worker-extension-cls", **parallel_kwargs["worker_extension_cls"]
        )
        parallel_group.add_argument(
            "--enable-fault-tolerance", **parallel_kwargs["enable_fault_tolerance"]
        )
        parallel_group.add_argument(
            "--fault-tolerance-config", **parallel_kwargs["fault_tolerance_config"]
        )

        # KV cache arguments
        cache_kwargs = get_kwargs(CacheConfig)
        cache_group = parser.add_argument_group(
            title="CacheConfig",
            description=CacheConfig.__doc__,
        )
        cache_group.add_argument("--block-size", **cache_kwargs["block_size"])
        cache_group.add_argument(
            "--gpu-memory-utilization",
            "--device-memory-utilization",
            dest="gpu_memory_utilization",
            **cache_kwargs["gpu_memory_utilization"],
        )
        cache_group.add_argument(
            "--kv-cache-memory-bytes", **cache_kwargs["kv_cache_memory_bytes"]
        )
        cache_group.add_argument("--kv-cache-dtype", **cache_kwargs["cache_dtype"])
        cache_group.add_argument(
            "--num-gpu-blocks-override", **cache_kwargs["num_gpu_blocks_override"]
        )
        cache_group.add_argument(
            "--enable-prefix-caching",
            **{
                **cache_kwargs["enable_prefix_caching"],
                "default": None,
            },
        )
        cache_group.add_argument(
            "--prefix-caching-hash-algo", **cache_kwargs["prefix_caching_hash_algo"]
        )
        cache_group.add_argument(
            "--prefix-cache-retention-interval",
            **cache_kwargs["prefix_cache_retention_interval"],
        )
        cache_group.add_argument(
            "--kv-cache-dtype-skip-layers", **cache_kwargs["kv_cache_dtype_skip_layers"]
        )
        cache_group.add_argument(
            "--kv-sharing-fast-prefill", **cache_kwargs["kv_sharing_fast_prefill"]
        )
        cache_group.add_argument(
            "--swa-bounded-replay", **cache_kwargs["swa_bounded_replay"]
        )
        cache_group.add_argument(
            "--mamba-cache-dtype", **cache_kwargs["mamba_cache_dtype"]
        )
        cache_group.add_argument(
            "--mamba-ssm-cache-dtype", **cache_kwargs["mamba_ssm_cache_dtype"]
        )
        cache_group.add_argument(
            "--mamba-block-size", **cache_kwargs["mamba_block_size"]
        )
        cache_group.add_argument(
            "--prefix-match-unit", **cache_kwargs["prefix_match_unit"]
        )
        cache_group.add_argument(
            "--mamba-cache-mode",
            **{
                **cache_kwargs["mamba_cache_mode"],
                "default": None,
                "help": _MAMBA_CACHE_MODE_DEPRECATION_MESSAGE,
            },
        )
        cache_group.add_argument(
            "--enable-mamba-shared-prefix-checkpoint",
            **cache_kwargs["enable_mamba_shared_prefix_checkpoint"],
        )
        cache_group.add_argument(
            "--replayssm-buffer-len", **cache_kwargs["replayssm_buffer_len"]
        )
        cache_group.add_argument("--use-replayssm", **cache_kwargs["use_replayssm"])
        cache_group.add_argument(
            "--kv-offloading-size", **cache_kwargs["kv_offloading_size"]
        )
        cache_group.add_argument(
            "--kv-offloading-backend", **cache_kwargs["kv_offloading_backend"]
        )

        # Model weight offload related configs
        offload_kwargs = get_kwargs(OffloadConfig)
        uva_kwargs = get_kwargs(UVAOffloadConfig)
        prefetch_kwargs = get_kwargs(PrefetchOffloadConfig)
        offload_group = parser.add_argument_group(
            title="OffloadConfig",
            description=OffloadConfig.__doc__,
        )
        offload_group.add_argument(
            "--offload-backend", **offload_kwargs["offload_backend"]
        )
        offload_group.add_argument("--cpu-offload-gb", **uva_kwargs["cpu_offload_gb"])
        offload_group.add_argument(
            "--cpu-offload-params", **uva_kwargs["cpu_offload_params"]
        )
        offload_group.add_argument(
            "--offload-group-size",
            **prefetch_kwargs["offload_group_size"],
        )
        offload_group.add_argument(
            "--offload-num-in-group",
            **prefetch_kwargs["offload_num_in_group"],
        )
        offload_group.add_argument(
            "--offload-prefetch-step",
            **prefetch_kwargs["offload_prefetch_step"],
        )
        offload_group.add_argument(
            "--offload-params", **prefetch_kwargs["offload_params"]
        )

        # Multimodal related configs
        multimodal_kwargs = get_kwargs(MultiModalConfig)
        multimodal_group = parser.add_argument_group(
            title="MultiModalConfig",
            description=MultiModalConfig.__doc__,
        )
        multimodal_group.add_argument(
            "--language-model-only", **multimodal_kwargs["language_model_only"]
        )
        multimodal_group.add_argument(
            "--limit-mm-per-prompt", **multimodal_kwargs["limit_per_prompt"]
        )
        multimodal_group.add_argument(
            "--enable-mm-embeds", **multimodal_kwargs["enable_mm_embeds"]
        )
        multimodal_group.add_argument(
            "--media-io-kwargs", **multimodal_kwargs["media_io_kwargs"]
        )
        multimodal_group.add_argument(
            "--mm-processor-kwargs", **multimodal_kwargs["mm_processor_kwargs"]
        )
        multimodal_group.add_argument(
            "--mm-processor-cache-gb", **multimodal_kwargs["mm_processor_cache_gb"]
        )
        multimodal_group.add_argument(
            "--mm-processor-cache-type", **multimodal_kwargs["mm_processor_cache_type"]
        )
        multimodal_group.add_argument(
            "--mm-hasher-algorithm", **multimodal_kwargs["mm_hasher_algorithm"]
        )
        multimodal_group.add_argument(
            "--mm-shm-cache-max-object-size-mb",
            **multimodal_kwargs["mm_shm_cache_max_object_size_mb"],
        )
        multimodal_group.add_argument(
            "--mm-encoder-only", **multimodal_kwargs["mm_encoder_only"]
        )
        multimodal_group.add_argument(
            "--mm-encoder-tp-mode", **multimodal_kwargs["mm_encoder_tp_mode"]
        )
        multimodal_group.add_argument(
            "--mm-encoder-attn-backend",
            **multimodal_kwargs["mm_encoder_attn_backend"],
        )
        multimodal_group.add_argument(
            "--mm-encoder-attn-dtype",
            **multimodal_kwargs["mm_encoder_attn_dtype"],
        )
        multimodal_group.add_argument(
            "--mm-encoder-fp8-scale-path",
            **multimodal_kwargs["mm_encoder_fp8_scale_path"],
        )
        multimodal_group.add_argument(
            "--mm-encoder-fp8-scale-save-path",
            **multimodal_kwargs["mm_encoder_fp8_scale_save_path"],
        )
        multimodal_group.add_argument(
            "--mm-encoder-fp8-scale-save-margin",
            **multimodal_kwargs["mm_encoder_fp8_scale_save_margin"],
        )
        multimodal_group.add_argument(
            "--interleave-mm-strings", **multimodal_kwargs["interleave_mm_strings"]
        )
        multimodal_group.add_argument(
            "--skip-mm-profiling", **multimodal_kwargs["skip_mm_profiling"]
        )

        multimodal_group.add_argument(
            "--video-pruning-rate", **multimodal_kwargs["video_pruning_rate"]
        )
        multimodal_group.add_argument(
            "--video-pruning-method",
            **multimodal_kwargs["video_pruning_method"],
        )
        multimodal_group.add_argument(
            "--mm-tensor-ipc", **multimodal_kwargs["mm_tensor_ipc"]
        )
        multimodal_group.add_argument(
            "--mm-processor-device",
            choices=["auto", "cpu"]
            + (
                [current_platform.device_type]
                if current_platform.device_type not in ("", "cpu")
                else []
            ),
            default="auto",
            help="Device the HF multi-modal processor runs the image/video "
            "transform on. Convenience for `--mm-processor-kwargs "
            "'{\"device\": ...}'`: the value is resolved here and stored there, "
            "it is not kept as separate state. Only takes effect for HF "
            '"fast" (torchvision-backed) processors, which accept a `device` '
            "argument; the others ignore it and stay on CPU.\n\n"
            '"auto" uses the accelerator on encoder instances of an '
            "encode/prefill/decode deployment -- an EC producer that is not "
            "also a consumer allocates no KV cache, so its accelerator is not "
            "contended by the language model -- and then only when "
            "`--mm-tensor-ipc=torch_shm` can carry device tensors, since every "
            "other transport would copy the result back to the host and that "
            'copy costs more than it saves. "auto" resolves to "cpu" '
            "everywhere else.",
        )
        multimodal_group.add_argument(
            "--mm-ipc-gpu-memory-gb",
            **multimodal_kwargs["mm_ipc_gpu_memory_gb"],
        )
        multimodal_group.add_argument(
            "--mm-device-do-normalize",
            **{
                **multimodal_kwargs["mm_device_do_normalize"],
                "default": None,
            },
        )

        # LoRA related configs
        lora_kwargs = get_kwargs(LoRAConfig)
        lora_group = parser.add_argument_group(
            title="LoRAConfig",
            description=LoRAConfig.__doc__,
        )
        lora_group.add_argument(
            "--enable-lora",
            action=argparse.BooleanOptionalAction,
            help="If True, enable handling of LoRA adapters.",
        )
        lora_group.add_argument("--max-loras", **lora_kwargs["max_loras"])
        lora_group.add_argument("--max-lora-rank", **lora_kwargs["max_lora_rank"])
        lora_group.add_argument(
            "--lora-dtype",
            **lora_kwargs["lora_dtype"],
        )
        lora_group.add_argument(
            "--enable-tower-connector-lora",
            **lora_kwargs["enable_tower_connector_lora"],
        )
        lora_group.add_argument("--max-cpu-loras", **lora_kwargs["max_cpu_loras"])
        lora_group.add_argument(
            "--fully-sharded-loras", **lora_kwargs["fully_sharded_loras"]
        )
        lora_group.add_argument(
            "--lora-target-modules", **lora_kwargs["target_modules"]
        )
        lora_group.add_argument("--default-mm-loras", **lora_kwargs["default_mm_loras"])
        lora_group.add_argument(
            "--specialize-active-lora", **lora_kwargs["specialize_active_lora"]
        )
        lora_group.add_argument(
            "--enable-mixed-moe-lora-format",
            **lora_kwargs["enable_mixed_moe_lora_format"],
        )
        lora_group.add_argument(
            "--enable-moe-shared-loras",
            **lora_kwargs["enable_moe_shared_loras"],
        )
        lora_group.add_argument(
            "--max-lora-cls-labels",
            **lora_kwargs["max_lora_cls_labels"],
        )

        # Logging arguments
        logging_group = parser.add_argument_group(
            title="LoggingConfig",
            description=LoggingConfig.__doc__,
        )
        logging_config_kwargs = get_kwargs(VllmConfig)["logging_config"]
        logging_config_kwargs["default"] = argparse.SUPPRESS
        logging_group.add_argument("--logging-config", **logging_config_kwargs)
        logging_group.add_argument(
            "--log-level",
            choices=get_args(LogLevel),
            default=argparse.SUPPRESS,
            help=(
                "Shortcut for --logging-config.log_level. "
                "Overrides that field if both are specified."
            ),
        )
        logging_group.add_argument(
            "--log-config-file",
            dest="log_config_file",
            default=argparse.SUPPRESS,
            metavar="PYLOGGING_CONFIG_FILE",
            action=DeprecatedLogConfigFileAction,
            help=_LOG_CONFIG_FILE_DEPRECATION_MESSAGE,
        )
        # Retain the warning in v0.31.0 and v0.32.0. Remove this option and its
        # compatibility mapping in create_logging_config() in v0.33.0.

        # Observability arguments
        observability_kwargs = get_kwargs(ObservabilityConfig)
        observability_group = parser.add_argument_group(
            title="ObservabilityConfig",
            description=ObservabilityConfig.__doc__,
        )
        observability_group.add_argument(
            "--show-hidden-metrics-for-version",
            **observability_kwargs["show_hidden_metrics_for_version"],
        )
        observability_group.add_argument(
            "--otlp-traces-endpoint", **observability_kwargs["otlp_traces_endpoint"]
        )
        observability_group.add_argument(
            "--collect-detailed-traces",
            **observability_kwargs["collect_detailed_traces"],
        )
        observability_group.add_argument(
            "--per-request-spec-decode-metrics",
            **observability_kwargs["per_request_spec_decode_metrics"],
        )
        observability_group.add_argument(
            "--kv-cache-metrics", **observability_kwargs["kv_cache_metrics"]
        )
        observability_group.add_argument(
            "--kv-cache-metrics-sample",
            **observability_kwargs["kv_cache_metrics_sample"],
        )
        observability_group.add_argument(
            "--custom-histogram-buckets",
            **observability_kwargs["custom_histogram_buckets"],
        )
        observability_group.add_argument(
            "--cudagraph-metrics",
            **observability_kwargs["cudagraph_metrics"],
        )
        observability_group.add_argument(
            "--enable-layerwise-nvtx-tracing",
            **observability_kwargs["enable_layerwise_nvtx_tracing"],
        )
        observability_group.add_argument(
            "--enable-mfu-metrics",
            **observability_kwargs["enable_mfu_metrics"],
        )
        observability_group.add_argument(
            "--enable-logging-iteration-details",
            **observability_kwargs["enable_logging_iteration_details"],
        )
        observability_group.add_argument(
            "--jit-monitor-mode",
            **observability_kwargs["jit_monitor_mode"],
        )
        observability_group.add_argument(
            "--jit-monitor-verbose",
            **observability_kwargs["jit_monitor_verbose"],
        )

        # Scheduler arguments
        scheduler_kwargs = get_kwargs(SchedulerConfig)
        scheduler_group = parser.add_argument_group(
            title="SchedulerConfig",
            description=SchedulerConfig.__doc__,
        )
        scheduler_group.add_argument(
            "--max-num-batched-tokens",
            **{
                **scheduler_kwargs["max_num_batched_tokens"],
                "default": None,
            },
        )
        scheduler_group.add_argument(
            "--max-num-scheduled-tokens",
            **{
                **scheduler_kwargs["max_num_scheduled_tokens"],
                "default": None,
            },
        )
        scheduler_group.add_argument(
            "--max-num-seqs",
            **{
                **scheduler_kwargs["max_num_seqs"],
                "default": None,
            },
        )
        scheduler_group.add_argument(
            "--max-num-active-seqs",
            **scheduler_kwargs["max_num_active_seqs"],
        )
        scheduler_group.add_argument(
            "--max-num-queued-reqs", **scheduler_kwargs["max_num_queued_reqs"]
        )
        scheduler_group.add_argument(
            "--max-num-queued-tokens",
            **scheduler_kwargs["max_num_queued_tokens"],
        )
        scheduler_group.add_argument(
            "--long-prefill-token-threshold",
            **scheduler_kwargs["long_prefill_token_threshold"],
        )
        scheduler_group.add_argument(
            "--long-prefill-token-threshold-adaptive",
            **scheduler_kwargs["long_prefill_token_threshold_adaptive"],
        )
        # multi-step scheduling has been removed; corresponding arguments
        # are no longer supported.
        scheduler_group.add_argument(
            "--scheduling-policy", **scheduler_kwargs["policy"]
        )
        scheduler_group.add_argument(
            "--enable-chunked-prefill",
            **{
                **scheduler_kwargs["enable_chunked_prefill"],
                "default": None,
            },
        )
        scheduler_group.add_argument(
            "--disable-chunked-mm-input", **scheduler_kwargs["disable_chunked_mm_input"]
        )
        scheduler_group.add_argument(
            "--scheduler-cls", **scheduler_kwargs["scheduler_cls"]
        )
        scheduler_group.add_argument(
            "--scheduler-reserve-full-isl",
            **scheduler_kwargs["scheduler_reserve_full_isl"],
        )
        scheduler_group.add_argument("--watermark", **scheduler_kwargs["watermark"])
        scheduler_group.add_argument(
            "--prefill-schedule-interval",
            **scheduler_kwargs["prefill_schedule_interval"],
        )
        scheduler_group.add_argument(
            "--disable-hybrid-kv-cache-manager",
            **scheduler_kwargs["disable_hybrid_kv_cache_manager"],
        )
        scheduler_group.add_argument(
            "--async-scheduling", **scheduler_kwargs["async_scheduling"]
        )
        scheduler_group.add_argument(
            "--stream-interval", **scheduler_kwargs["stream_interval"]
        )

        # Compilation arguments
        compilation_kwargs = get_kwargs(CompilationConfig)
        compilation_group = parser.add_argument_group(
            title="CompilationConfig",
            description=CompilationConfig.__doc__,
        )
        compilation_group.add_argument(
            "--cudagraph-capture-sizes", **compilation_kwargs["cudagraph_capture_sizes"]
        )
        compilation_group.add_argument(
            "--max-cudagraph-capture-size",
            **compilation_kwargs["max_cudagraph_capture_size"],
        )

        # Kernel arguments
        kernel_kwargs = get_kwargs(KernelConfig)
        kernel_group = parser.add_argument_group(
            title="KernelConfig",
            description=KernelConfig.__doc__,
        )
        kernel_group.add_argument("--ir-op-priority", **kernel_kwargs["ir_op_priority"])
        kernel_group.add_argument(
            "--enable-flashinfer-autotune",
            **kernel_kwargs["enable_flashinfer_autotune"],
        )
        kernel_group.add_argument("--monokernel", **kernel_kwargs["monokernel"])
        moe_backend_kwargs = kernel_kwargs["moe_backend"]
        moe_backend_kwargs["type"] = lambda s: s.lower().replace("-", "_")
        kernel_group.add_argument("--moe-backend", **moe_backend_kwargs)
        linear_backend_kwargs = kernel_kwargs["linear_backend"]
        linear_backend_kwargs["type"] = lambda s: s.lower().replace("-", "_")
        kernel_group.add_argument("--linear-backend", **linear_backend_kwargs)
        sparse_indexer_topk_kwargs = kernel_kwargs["sparse_indexer_topk_backend"]
        sparse_indexer_topk_kwargs["type"] = lambda s: s.lower().replace("-", "_")
        kernel_group.add_argument(
            "--sparse-indexer-topk-backend", **sparse_indexer_topk_kwargs
        )

        # vLLM arguments
        vllm_kwargs = get_kwargs(VllmConfig)
        vllm_group = parser.add_argument_group(
            title="VllmConfig",
            description=VllmConfig.__doc__,
        )
        # We construct SpeculativeConfig using fields from other configs in
        # create_engine_config. So we set the type to a JSON string here to
        # delay the Pydantic validation that comes with SpeculativeConfig.
        vllm_kwargs["speculative_config"]["type"] = optional_type(json.loads)
        vllm_group.add_argument(
            "--speculative-config", "-sc", **vllm_kwargs["speculative_config"]
        )
        speculative_kwargs = get_kwargs(SpeculativeConfig)
        vllm_group.add_argument("--spec-method", **speculative_kwargs["method"])
        vllm_group.add_argument("--spec-model", **speculative_kwargs["model"])
        vllm_group.add_argument(
            "--spec-tokens", **speculative_kwargs["num_speculative_tokens"]
        )
        vllm_kwargs["watermark_config"]["type"] = optional_type(json.loads)
        vllm_group.add_argument("--watermark-config", **vllm_kwargs["watermark_config"])
        vllm_kwargs["diffusion_config"]["type"] = optional_type(json.loads)
        vllm_group.add_argument(
            "--diffusion-config", "-dc", **vllm_kwargs["diffusion_config"]
        )
        vllm_group.add_argument(
            "--kv-transfer-config", **vllm_kwargs["kv_transfer_config"]
        )
        vllm_group.add_argument("--kv-events-config", **vllm_kwargs["kv_events_config"])
        vllm_group.add_argument(
            "--ec-transfer-config", **vllm_kwargs["ec_transfer_config"]
        )
        vllm_group.add_argument(
            "--ec-manager-config", **vllm_kwargs["ec_manager_config"]
        )
        vllm_group.add_argument(
            "--compilation-config", "-cc", **vllm_kwargs["compilation_config"]
        )
        vllm_group.add_argument(
            "--attention-config", "-ac", **vllm_kwargs["attention_config"]
        )
        vllm_group.add_argument("--engram-config", **vllm_kwargs["engram_config"])
        vllm_group.add_argument("--reasoning-config", **vllm_kwargs["reasoning_config"])
        vllm_group.add_argument("--kernel-config", **vllm_kwargs["kernel_config"])
        vllm_group.add_argument(
            "--additional-config", **vllm_kwargs["additional_config"]
        )
        vllm_group.add_argument(
            "--structured-outputs-config", **vllm_kwargs["structured_outputs_config"]
        )
        vllm_group.add_argument(
            "--aux-output-config", **vllm_kwargs["aux_output_config"]
        )
        vllm_group.add_argument("--profiler-config", **vllm_kwargs["profiler_config"])
        vllm_group.add_argument(
            "--optimization-level", **vllm_kwargs["optimization_level"]
        )
        vllm_group.add_argument("--performance-mode", **vllm_kwargs["performance_mode"])
        vllm_group.add_argument(
            "--weight-transfer-config", **vllm_kwargs["weight_transfer_config"]
        )

        # Other arguments
        parser.add_argument(
            "--disable-log-stats",
            action="store_true",
            help="Disable logging statistics.",
        )

        parser.add_argument(
            "--aggregate-engine-logging",
            action="store_true",
            help="Log aggregate rather than per-engine statistics "
            "when using data parallelism.",
        )

        parser.add_argument(
            "--fail-on-environ-validation",
            help="If set, the engine will raise an error if "
            "environment validation fails.",
            default=False,
            action=argparse.BooleanOptionalAction,
        )

        parser.add_argument(
            "--shutdown-timeout",
            type=int,
            default=0,
            help="Shutdown timeout in seconds. 0 = abort, >0 = wait.",
        )

        parser.add_argument(
            "--gdn-prefill-backend",
            dest="gdn_prefill_backend",
            choices=["flashinfer", "triton", "cutedsl", "aiter_flydsl"],
            default=None,
            help="Select GDN prefill backend.",
        )
        parser.add_argument(
            "--kda-prefill-backend",
            dest="kda_prefill_backend",
            choices=["auto", "triton", "flashkda", "flashinfer", "fused"],
            default=None,
            help="Select KDA prefill backend. 'flashkda' is CUDA-only and "
            "'fused' is ROCm-only; 'auto' picks a supported backend.",
        )
        parser.add_argument(
            "--kda-decode-backend",
            dest="kda_decode_backend",
            choices=["auto", "native", "flashinfer", "triton"],
            default=None,
            help="Select KDA decode backend.",
        )
        return parser

    @classmethod
    def from_cli_args(cls, args: argparse.Namespace):
        # Get the list of attributes of this dataclass.
        attrs = [attr.name for attr in dataclasses.fields(cls)]

        # Set the attributes from the parsed arguments.
        engine_args = cls(
            **{attr: getattr(args, attr) for attr in attrs if hasattr(args, attr)}
        )
        return engine_args

    def create_model_config(self) -> ModelConfig:
        if not envs.VLLM_ENABLE_V1_MULTIPROCESSING:
            logger.warning(
                "The global random seed is set to %d. Since "
                "VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may "
                "affect the random state of the Python process that "
                "launched vLLM.",
                self.seed,
            )
        if self.disable_cascade_attn is not None:
            warnings.warn(_DISABLE_CASCADE_ATTN_DEPRECATION_MESSAGE, stacklevel=1)

        return ModelConfig(
            model=self.model,
            model_weights=self.model_weights,
            hf_config_path=self.hf_config_path,
            runner=self.runner,
            convert=self.convert,
            tokenizer=self.tokenizer,  # type: ignore[arg-type]
            tokenizer_mode=self.tokenizer_mode,
            trust_remote_code=self.trust_remote_code,
            allowed_local_media_path=self.allowed_local_media_path,
            allowed_media_domains=self.allowed_media_domains,
            dtype=self.dtype,
            seed=self.seed,
            revision=self.revision,
            code_revision=self.code_revision,
            hf_token=self.hf_token,
            hf_overrides=self.hf_overrides,
            model_class_overrides=self.model_class_overrides,
            tokenizer_revision=self.tokenizer_revision,
            max_model_len=self.max_model_len,
            quantization=self.quantization,
            quantization_config=self.quantization_config,
            allow_deprecated_quantization=self.allow_deprecated_quantization,
            enforce_eager=self.enforce_eager,
            return_sampling_mask=self.return_sampling_mask,
            max_logprobs=self.max_logprobs,
            logprobs_mode=self.logprobs_mode,
            use_fp64_gumbel=self.use_fp64_gumbel,
            enable_trace_replay=self.enable_trace_replay,
            disable_sliding_window=self.disable_sliding_window,
            skip_tokenizer_init=self.skip_tokenizer_init,
            enable_prompt_embeds=self.enable_prompt_embeds,
            served_model_name=self.served_model_name,
            language_model_only=self.language_model_only,
            limit_mm_per_prompt=self.limit_mm_per_prompt,
            enable_mm_embeds=self.enable_mm_embeds,
            interleave_mm_strings=self.interleave_mm_strings,
            media_io_kwargs=self.media_io_kwargs,
            skip_mm_profiling=self.skip_mm_profiling,
            config_format=self.config_format,
            mm_processor_kwargs=self.mm_processor_kwargs,
            mm_processor_cache_gb=self.mm_processor_cache_gb,
            mm_processor_cache_type=self.mm_processor_cache_type,
            mm_shm_cache_max_object_size_mb=self.mm_shm_cache_max_object_size_mb,
            mm_hasher_algorithm=self.mm_hasher_algorithm,
            mm_encoder_only=self.mm_encoder_only,
            mm_encoder_tp_mode=self.mm_encoder_tp_mode,
            mm_encoder_attn_backend=self.mm_encoder_attn_backend,
            mm_encoder_attn_dtype=self.mm_encoder_attn_dtype,
            mm_encoder_fp8_scale_path=self.mm_encoder_fp8_scale_path,
            mm_encoder_fp8_scale_save_path=self.mm_encoder_fp8_scale_save_path,
            mm_encoder_fp8_scale_save_margin=self.mm_encoder_fp8_scale_save_margin,
            pooler_config=self.pooler_config,
            generation_config=self.generation_config,
            override_generation_config=self.override_generation_config,
            enable_sleep_mode=self.enable_sleep_mode,
            sleep_mode_offload_cudagraph=self.sleep_mode_offload_cudagraph,
            sleep_preserve_parameter_names=self.sleep_preserve_parameter_names,
            enable_cumem_allocator=self.enable_cumem_allocator,
            enable_nccl_comm_suspend=self.enable_nccl_comm_suspend,
            model_impl=self.model_impl,
            logits_processors=self.logits_processors,
            video_pruning_rate=self.video_pruning_rate,
            video_pruning_method=self.video_pruning_method,
            mm_tensor_ipc=self.mm_tensor_ipc,
            mm_ipc_gpu_memory_gb=self.mm_ipc_gpu_memory_gb,
            mm_device_do_normalize=self.mm_device_do_normalize,
            mm_processor_device=self.mm_processor_device,
            io_processor_plugin=self.io_processor_plugin,
            renderer_num_workers=self.renderer_num_workers,
        )

    def validate_tensorizer_args(self):
        from vllm.model_executor.model_loader.tensorizer import TensorizerConfig

        for key in self.model_loader_extra_config:
            if key in TensorizerConfig._fields:
                self.model_loader_extra_config["tensorizer_config"][key] = (
                    self.model_loader_extra_config[key]
                )

    def create_load_config(self) -> LoadConfig:
        if self.load_format == "tensorizer":
            if hasattr(self.model_loader_extra_config, "to_serializable"):
                self.model_loader_extra_config = (
                    self.model_loader_extra_config.to_serializable()
                )
            self.model_loader_extra_config["tensorizer_config"] = {}
            self.model_loader_extra_config["tensorizer_config"]["tensorizer_dir"] = (
                self.model
            )
            self.validate_tensorizer_args()

        return LoadConfig(
            load_format=self.load_format,
            download_dir=self.download_dir,
            safetensors_load_strategy=self.safetensors_load_strategy,
            safetensors_prefetch_num_threads=self.safetensors_prefetch_num_threads,
            safetensors_prefetch_block_size=self.safetensors_prefetch_block_size,
            model_loader_extra_config=self.model_loader_extra_config,
            ignore_patterns=self.ignore_patterns,
            use_tqdm_on_load=self.use_tqdm_on_load,
            pt_load_map_location=self.pt_load_map_location,
        )

    def create_speculative_config(
        self,
        target_model_config: ModelConfig,
        target_parallel_config: ParallelConfig,
    ) -> SpeculativeConfig | None:
        """Initializes and returns a SpeculativeConfig object based on
        `speculative_config`.
        """
        for flag, key, value in (
            ("--spec-method", "method", self.spec_method),
            ("--spec-model", "model", self.spec_model),
            ("--spec-tokens", "num_speculative_tokens", self.spec_tokens),
        ):
            if value is None:
                continue
            if self.speculative_config is None:
                self.speculative_config = {}
            if key in self.speculative_config:
                raise ValueError(
                    f"{flag} and --speculative-config['{key}'] are mutually exclusive"
                )
            self.speculative_config[key] = value

        if self.speculative_config is None:
            return None

        self.speculative_config = {
            k.replace("-", "_"): v for k, v in self.speculative_config.items()
        }

        # Note(Shangming): These parameters are not obtained from the cli arg
        # '--speculative-config' and must be passed in when creating the engine
        # config.
        self.speculative_config.update(
            {
                "target_model_config": target_model_config,
                "target_parallel_config": target_parallel_config,
            }
        )
        return SpeculativeConfig(**self.speculative_config)

    def _resolve_device_ids(self) -> list[int] | None:
        if not self.device_ids:
            return None
        if self.distributed_executor_backend == "ray":
            logger.warning(
                "--device-ids has no effect when using the Ray executor. "
                "Use Ray placement groups for GPU selection instead."
            )
        ids = self.device_ids
        if len(set(ids)) != len(ids):
            raise ValueError(f"--device-ids must not contain duplicates: {ids}")
        if all(isinstance(i, str) for i in ids):
            return [
                current_platform.device_control_id_to_physical_device_id(i)
                for i in cast(list[str], ids)
            ]
        if any(isinstance(i, str) for i in ids):
            raise ValueError("--device-ids must not mix integer IDs and UUIDs")
        int_ids = cast(list[int], ids)
        # Compose with CUDA_VISIBLE_DEVICES: if CVD is set, treat
        # --device-ids values as indices into the CVD-visible set.
        cvd = getattr(
            envs,
            current_platform.device_control_env_var,
            os.environ.get(current_platform.device_control_env_var),
        )
        if cvd:
            cvd_ids = [
                current_platform.device_control_id_to_physical_device_id(x)
                for x in cvd.split(",")
            ]
            for i in int_ids:
                if not 0 <= i < len(cvd_ids):
                    raise ValueError(
                        f"--device-ids index {i} is out of range for "
                        f"{current_platform.device_control_env_var}"
                        f"={cvd} ({len(cvd_ids)} devices visible)"
                    )
            return [cvd_ids[i] for i in int_ids]
        return int_ids

    def create_diffusion_config(self) -> DiffusionConfig | None:
        if self.diffusion_config is None:
            return None
        cfg = self.diffusion_config
        if isinstance(cfg, str):
            cfg = json.loads(cfg)
        return DiffusionConfig(**cfg)

    def create_watermark_config(self) -> WatermarkConfig | None:
        if self.watermark_config is None:
            return None
        cfg = self.watermark_config
        if isinstance(cfg, str):
            cfg = json.loads(cfg)
        return WatermarkConfig(**cfg)

    def create_structured_outputs_config(self) -> StructuredOutputsConfig:
        """Merge frontend parser flags into the structured outputs config.

        Mutates `self.structured_outputs_config` in place and returns it
        (not a copy). Model-specific defaults (e.g. gpt_oss ->
        "openai_gptoss") are applied later by `verify_and_update_config`
        only when the resolved value is still empty, so explicit CLI flags
        take precedence.
        """
        if self.reasoning_parser:
            self.structured_outputs_config.reasoning_parser = self.reasoning_parser

        if self.reasoning_parser_plugin:
            self.structured_outputs_config.reasoning_parser_plugin = (
                self.reasoning_parser_plugin
            )

        return self.structured_outputs_config

    def create_observability_config(self) -> ObservabilityConfig:
        return ObservabilityConfig(
            show_hidden_metrics_for_version=self.show_hidden_metrics_for_version,
            otlp_traces_endpoint=self.otlp_traces_endpoint,
            collect_detailed_traces=self.collect_detailed_traces,
            per_request_spec_decode_metrics=self.per_request_spec_decode_metrics,
            kv_cache_metrics=self.kv_cache_metrics,
            kv_cache_metrics_sample=self.kv_cache_metrics_sample,
            custom_histogram_buckets=self.custom_histogram_buckets,
            cudagraph_metrics=self.cudagraph_metrics,
            enable_layerwise_nvtx_tracing=self.enable_layerwise_nvtx_tracing,
            enable_mfu_metrics=self.enable_mfu_metrics,
            enable_mm_processor_stats=self.enable_mm_processor_stats,
            enable_logging_iteration_details=self.enable_logging_iteration_details,
            jit_monitor_mode=self.jit_monitor_mode,
            jit_monitor_verbose=self.jit_monitor_verbose,
        )

    def create_logging_config(self) -> LoggingConfig:
        config = self.logging_config or LoggingConfig()
        if self.log_level is not None:
            config = dataclasses.replace(config, log_level=self.log_level)
        if self.log_config_file is not None:
            config = dataclasses.replace(
                config, pylogging_config_file=self.log_config_file
            )
        return config

    def create_engine_config(
        self,
        usage_context: UsageContext | None = None,
        headless: bool = False,
    ) -> VllmConfig:
        """Create the VllmConfig.

        NOTE: If VllmConfig is incompatible, we raise an error.
        """
        logging_config = self.create_logging_config()
        configure_logging_if_needed(logging_config)

        current_platform.pre_register_and_update()

        device_config = DeviceConfig(device=cast(Device, current_platform.device_type))

        current_platform.validate_environ(self.fail_on_environ_validation)

        # Check if the model is a speculator and override model/tokenizer/config
        # BEFORE creating ModelConfig, so the config is created with the target model
        # Skip speculator detection for cloud storage models (eg: S3, GCS) since
        # HuggingFace cannot load configs directly from S3 URLs. S3 models can still
        # use speculators with explicit --speculative-config.
        if not is_cloud_storage(self.model):
            (self.model, self.tokenizer, self.speculative_config) = (
                maybe_override_with_speculators(
                    model=self.model,
                    tokenizer=self.tokenizer,
                    revision=self.revision,
                    trust_remote_code=self.trust_remote_code,
                    vllm_speculative_config=self.speculative_config,
                    hf_token=self.hf_token,
                )
            )

        model_config = self.create_model_config()
        self.model = model_config.model
        self.model_weights = model_config.model_weights
        self.tokenizer = model_config.tokenizer

        self._check_feature_supported()
        self._set_default_chunked_prefill_and_prefix_caching_args(model_config)
        self._set_default_reasoning_config_args()
        sliding_window: int | None = None
        layer_types = getattr(model_config.hf_text_config, "layer_types", None)
        if layer_types is None or all(lt == "sliding_attention" for lt in layer_types):
            # Only set CacheConfig.sliding_window if the model is all sliding
            # window. Otherwise CacheConfig.sliding_window will override the
            # global layers in interleaved sliding window models.
            sliding_window = model_config.get_sliding_window()

        # Resolve "auto" kv_cache_dtype to actual value from model config
        resolved_cache_dtype = resolve_kv_cache_dtype_string(
            self.kv_cache_dtype, model_config
        )

        assert self.enable_prefix_caching is not None, (
            "enable_prefix_caching must be set by this point"
        )
        if self.mamba_cache_mode is not None:
            logger.warning_once(_MAMBA_CACHE_MODE_DEPRECATION_MESSAGE)

        cache_config = CacheConfig(
            block_size=self.block_size,  # type: ignore[arg-type]
            gpu_memory_utilization=self.gpu_memory_utilization,
            kv_cache_memory_bytes=self.kv_cache_memory_bytes,
            cache_dtype=resolved_cache_dtype,  # type: ignore[arg-type]
            is_attention_free=model_config.is_attention_free,
            num_gpu_blocks_override=self.num_gpu_blocks_override,
            sliding_window=sliding_window,
            enable_prefix_caching=self.enable_prefix_caching,
            prefix_caching_hash_algo=self.prefix_caching_hash_algo,
            prefix_cache_retention_interval=self.prefix_cache_retention_interval,
            kv_cache_dtype_skip_layers=self.kv_cache_dtype_skip_layers,
            kv_sharing_fast_prefill=self.kv_sharing_fast_prefill,
            swa_bounded_replay=self.swa_bounded_replay,
            mamba_cache_dtype=self.mamba_cache_dtype,
            mamba_ssm_cache_dtype=self.mamba_ssm_cache_dtype,
            mamba_block_size=self.mamba_block_size,
            prefix_match_unit=self.prefix_match_unit,
            enable_mamba_shared_prefix_checkpoint=(
                self.enable_mamba_shared_prefix_checkpoint
            ),
            replayssm_buffer_len=self.replayssm_buffer_len,
            use_replayssm=self.use_replayssm,
            kv_offloading_size=self.kv_offloading_size,
            kv_offloading_backend=self.kv_offloading_backend,
        )

        # TurboQuant and UltraQuant both keep boundary attention layers at the
        # native dtype, and compute those layers the same way.
        uses_packed_kv_backend = (
            resolved_cache_dtype.startswith("turboquant_")
            or resolved_cache_dtype == "ultraquant_4bit"
        )
        if uses_packed_kv_backend:
            from vllm.model_executor.layers.quantization.turboquant.config import (
                TurboQuantConfig,
            )

            boundary = TurboQuantConfig.get_boundary_skip_layers(model_config)
            # Sorted so the order does not depend on set iteration: the list is
            # a CacheConfig hash factor. Entries can also be attention type
            # names (e.g. "sliding_window"), so sort them as strings.
            cache_config.kv_cache_dtype_skip_layers = sorted(
                set(cache_config.kv_cache_dtype_skip_layers) | set(boundary)
            )

        ray_runtime_env = None
        if is_ray_initialized():
            # Ray Serve LLM calls `create_engine_config` in the context
            # of a Ray task, therefore we check is_ray_initialized()
            # as opposed to is_in_ray_actor().
            import ray

            ray_runtime_env = ray.get_runtime_context().runtime_env
            # Avoid logging sensitive environment variables
            sanitized_env = ray_runtime_env.to_dict() if ray_runtime_env else {}
            if "env_vars" in sanitized_env:
                sanitized_env["env_vars"] = {
                    k: "***" for k in sanitized_env["env_vars"]
                }
            logger.info("Using ray runtime env (env vars redacted): %s", sanitized_env)

        # Get the current placement group if Ray is initialized and
        # we are in a Ray actor. If so, then the placement group will be
        # passed to spawned processes.
        placement_group = None
        if is_in_ray_actor():
            import ray

            # This call initializes Ray automatically if it is not initialized,
            # but we should not do this here.
            placement_group = ray.util.get_current_placement_group()

        assert not headless or not self.data_parallel_hybrid_lb, (
            "data_parallel_hybrid_lb is not applicable in headless mode"
        )
        if self.data_parallel_hybrid_lb and self.data_parallel_external_lb:
            raise ValueError(
                "Invalid data-parallel launch options: "
                "`--data-parallel-hybrid-lb` and "
                "`--data-parallel-external-lb` cannot be enabled together. "
                "Enable only one load-balancing mode."
            )
        if self.nnodes > 1 and self.data_parallel_backend != "mp":
            raise ValueError(
                "Invalid data-parallel launch options: "
                f"`--nnodes {self.nnodes}` requires "
                "`--data-parallel-backend mp`; got "
                f"`--data-parallel-backend {self.data_parallel_backend}`. "
                "Use the MP backend or set `--nnodes 1`."
            )
        inferred_data_parallel_rank = 0
        if self.nnodes > 1:
            world_size_within_dp = (
                self.pipeline_parallel_size
                * self.tensor_parallel_size
                * self.prefill_context_parallel_size
            )
            world_size = self.data_parallel_size * world_size_within_dp
            if world_size % self.nnodes != 0:
                raise ValueError(
                    "Invalid data-parallel launch options: "
                    f"`--nnodes {self.nnodes}` must evenly divide the total "
                    f"world size ({world_size}). Adjust `--nnodes`, "
                    "`--data-parallel-size`, `--pipeline-parallel-size`, "
                    "`--tensor-parallel-size`, or `--prefill-context-parallel-size`."
                )
            if not 0 <= self.node_rank < self.nnodes:
                raise ValueError(
                    "Invalid data-parallel launch options: `--node-rank` must "
                    f"be between 0 and {self.nnodes - 1}; got "
                    f"`--node-rank {self.node_rank}`. Set it to this node's "
                    "zero-based index."
                )
            local_world_size = world_size // self.nnodes
            inferred_data_parallel_rank = (
                self.node_rank * local_world_size
            ) // world_size_within_dp
            if self.data_parallel_size > 1 and self.data_parallel_external_lb:
                if self.data_parallel_rank is None:
                    if self.nnodes % self.data_parallel_size != 0:
                        raise ValueError(
                            "Invalid data-parallel launch options: "
                            "`--node-rank` cannot unambiguously identify external "
                            "data-parallel ranks when `--nnodes` is not divisible "
                            "by `--data-parallel-size`. Set a unique "
                            "`--data-parallel-rank` for each external-LB process."
                        )
                    self.data_parallel_rank = inferred_data_parallel_rank
                    logger.info(
                        "Inferred data_parallel_rank %d from node_rank %d "
                        "for external lb",
                        self.data_parallel_rank,
                        self.node_rank,
                    )
            elif self.data_parallel_size_local is None:
                # Infer data parallel size local for internal dplb:
                self.data_parallel_size_local = max(
                    local_world_size // world_size_within_dp, 1
                )
        data_parallel_external_lb = (
            self.data_parallel_external_lb or self.data_parallel_rank is not None
        )
        if self.enable_fault_tolerance and not data_parallel_external_lb:
            raise ValueError(
                "Fault tolerance requires external load balancer mode "
                "(--data-parallel-external-lb or --data-parallel-rank). "
                "Internal LB mode is not supported."
            )
        if (
            self.data_parallel_size > 1
            and data_parallel_external_lb
            and not model_config.is_moe
        ):
            raise ValueError(
                "Non-MoE models do not support external data parallel mode. "
                "For external load balancing, launch independent vLLM "
                "instances without --data-parallel-* arguments."
            )
        # Local DP rank = 1, use pure-external LB.
        if data_parallel_external_lb:
            if self.data_parallel_rank is None:
                raise ValueError(
                    "Invalid data-parallel launch options: "
                    "`--data-parallel-external-lb` requires a data-parallel "
                    "rank. Set `--data-parallel-rank`, or set "
                    "`--data-parallel-size` greater than 1 and use `--nnodes` "
                    "with `--node-rank` so the rank can be inferred."
                )
            if self.data_parallel_size_local not in (1, None):
                raise ValueError(
                    "Invalid data-parallel launch options: an external "
                    "data-parallel rank requires `--data-parallel-size-local "
                    f"1`; got {self.data_parallel_size_local}. Set it to 1 or "
                    "omit it."
                )
            data_parallel_size_local = 1
            # Use full external lb if we have local_size of 1.
            self.data_parallel_hybrid_lb = False
        elif self.data_parallel_size_local is not None:
            data_parallel_size_local = self.data_parallel_size_local

            if self.data_parallel_start_rank is not None and not headless:
                # Infer hybrid LB mode.
                self.data_parallel_hybrid_lb = True

            if self.data_parallel_hybrid_lb and data_parallel_size_local == 1:
                # Use full external lb if we have local_size of 1.
                logger.warning(
                    "data_parallel_hybrid_lb is not eligible when "
                    "data_parallel_size_local = 1, autoswitch to "
                    "data_parallel_external_lb."
                )
                data_parallel_external_lb = True
                self.data_parallel_hybrid_lb = False

            if data_parallel_size_local == self.data_parallel_size:
                # Disable hybrid LB mode if set for a single node
                self.data_parallel_hybrid_lb = False

            self.data_parallel_rank = (
                self.data_parallel_start_rank
                if self.data_parallel_start_rank is not None
                else inferred_data_parallel_rank
            )
            if self.nnodes > 1:
                logger.info(
                    "Inferred data_parallel_rank %d from node_rank %d",
                    self.data_parallel_rank,
                    self.node_rank,
                )
        else:
            if self.data_parallel_hybrid_lb:
                raise ValueError(
                    "Invalid data-parallel launch options: "
                    "`--data-parallel-hybrid-lb` requires "
                    "`--data-parallel-size-local`. Set it to the number of "
                    "data-parallel ranks on this node."
                )

            if self.data_parallel_backend == "ray" and (
                envs.VLLM_RAY_DP_PACK_STRATEGY == "span"
            ):
                # Data parallel size defaults to 1 if DP ranks are spanning
                # multiple nodes
                data_parallel_size_local = 1
            else:
                # Otherwise local DP size defaults to global DP size if not set
                data_parallel_size_local = self.data_parallel_size

        # DP address, used in multi-node case for torch distributed group
        # and ZMQ sockets.
        if self.data_parallel_address is None:
            if self.data_parallel_backend == "ray":
                host_ip = get_ip()
                logger.info(
                    "Using host IP %s as ray-based data parallel address", host_ip
                )
                data_parallel_address = host_ip
            else:
                assert self.data_parallel_backend == "mp", (
                    "data_parallel_backend can only be ray or mp, got %s",
                    self.data_parallel_backend,
                )
                data_parallel_address = (
                    self.master_addr or ParallelConfig.data_parallel_master_ip
                )
        else:
            data_parallel_address = self.data_parallel_address

        # This port is only used when there are remote data parallel engines,
        # otherwise the local IPC transport is used.
        data_parallel_rpc_port = (
            self.data_parallel_rpc_port
            if (self.data_parallel_rpc_port is not None)
            else ParallelConfig.data_parallel_rpc_port
        )

        if self.tokens_only and not model_config.skip_tokenizer_init:
            model_config.skip_tokenizer_init = True
            logger.info("Skipping tokenizer initialization for tokens-only mode.")

        parallel_config = ParallelConfig(
            pipeline_parallel_size=self.pipeline_parallel_size,
            tensor_parallel_size=self.tensor_parallel_size,
            prefill_context_parallel_size=self.prefill_context_parallel_size,
            data_parallel_size=self.data_parallel_size,
            data_parallel_rank=self.data_parallel_rank or 0,
            data_parallel_external_lb=data_parallel_external_lb,
            data_parallel_size_local=data_parallel_size_local,
            master_addr=self.master_addr,
            master_port=self.master_port,
            nnodes=self.nnodes,
            node_rank=self.node_rank,
            distributed_timeout_seconds=self.distributed_timeout_seconds,
            cpu_distributed_timeout_seconds=self.cpu_distributed_timeout_seconds,
            data_parallel_master_ip=data_parallel_address,
            data_parallel_rpc_port=data_parallel_rpc_port,
            data_parallel_backend=self.data_parallel_backend,
            data_parallel_hybrid_lb=self.data_parallel_hybrid_lb,
            is_moe_model=model_config.is_moe,
            enable_hc_sp=self.enable_hc_sp,
            enable_expert_parallel=self.enable_expert_parallel,
            enable_batch_sharded_sampling=self.enable_batch_sharded_sampling,
            enable_ep_weight_filter=self.enable_ep_weight_filter,
            all2all_backend=self.all2all_backend,
            enable_elastic_ep=self.enable_elastic_ep,
            elastic_ep_max_dp_size=self.elastic_ep_max_dp_size,
            enable_dbo=self.enable_dbo,
            enable_shm_tensor_arena=self.enable_shm_tensor_arena,
            ubatch_size=self.ubatch_size,
            dbo_decode_token_threshold=self.dbo_decode_token_threshold,
            dp_sync_interval=self.dp_sync_interval,
            dbo_prefill_token_threshold=self.dbo_prefill_token_threshold,
            disable_nccl_for_dp_synchronization=self.disable_nccl_for_dp_synchronization,
            enable_eplb=self.enable_eplb,
            eplb_config=self.eplb_config,
            expert_placement_strategy=self.expert_placement_strategy,
            max_parallel_loading_workers=self.max_parallel_loading_workers,
            disable_custom_all_reduce=self.disable_custom_all_reduce,
            ray_workers_use_nsight=self.ray_workers_use_nsight,
            ray_runtime_env=ray_runtime_env,
            placement_group=placement_group,
            distributed_executor_backend=self.distributed_executor_backend,
            worker_cls=self.worker_cls,
            worker_extension_cls=self.worker_extension_cls,
            decode_context_parallel_size=self.decode_context_parallel_size,
            dcp_comm_backend=self.dcp_comm_backend,
            dcp_q_replicate=self.dcp_q_replicate,
            dcp_kv_cache_interleave_size=self.dcp_kv_cache_interleave_size,
            cp_kv_cache_interleave_size=(
                self.cp_kv_cache_interleave_size
                if self.cp_kv_cache_interleave_size is not None
                else ParallelConfig.cp_kv_cache_interleave_size
            ),
            _allow_auto_resolve_cp_interleave_size=(
                self.cp_kv_cache_interleave_size is None
            ),
            _api_process_count=self._api_process_count,
            _api_process_rank=self._api_process_rank,
            assigned_physical_gpu_ids=self._resolve_device_ids(),
            enable_fault_tolerance=self.enable_fault_tolerance,
            fault_tolerance_config=self.fault_tolerance_config,
            numa_bind=self.numa_bind,
            numa_bind_nodes=self.numa_bind_nodes,
            numa_bind_cpus=self.numa_bind_cpus,
        )

        speculative_config = self.create_speculative_config(
            target_model_config=model_config,
            target_parallel_config=parallel_config,
        )
        diffusion_config = self.create_diffusion_config()
        watermark_config = self.create_watermark_config()

        self._set_default_max_num_seqs_and_batched_tokens_args(
            usage_context,
            model_config,
            parallel_config,
        )

        assert self.max_num_batched_tokens is not None, (
            "max_num_batched_tokens must be set by this point"
        )
        assert self.max_num_seqs is not None, "max_num_seqs must be set by this point"
        assert self.enable_chunked_prefill is not None, (
            "enable_chunked_prefill must be set by this point"
        )
        assert model_config.max_model_len is not None, (
            "max_model_len must be set by this point"
        )
        scheduler_config = SchedulerConfig(
            runner_type=model_config.runner_type,
            max_num_batched_tokens=self.max_num_batched_tokens,
            max_num_scheduled_tokens=self.max_num_scheduled_tokens,
            max_num_seqs=self.max_num_seqs,
            max_num_active_seqs=self.max_num_active_seqs,
            max_num_queued_reqs=self.max_num_queued_reqs,
            max_num_queued_tokens=self.max_num_queued_tokens,
            max_model_len=model_config.max_model_len,
            enable_chunked_prefill=self.enable_chunked_prefill,
            disable_chunked_mm_input=self.disable_chunked_mm_input,
            is_multimodal_model=model_config.is_multimodal_model,
            is_encoder_decoder=model_config.is_encoder_decoder,
            policy=self.scheduling_policy,
            scheduler_cls=self.scheduler_cls,
            long_prefill_token_threshold=self.long_prefill_token_threshold,
            long_prefill_token_threshold_adaptive=(
                self.long_prefill_token_threshold_adaptive
            ),
            scheduler_reserve_full_isl=self.scheduler_reserve_full_isl,
            watermark=self.watermark,
            prefill_schedule_interval=self.prefill_schedule_interval,
            disable_hybrid_kv_cache_manager=self.disable_hybrid_kv_cache_manager,
            async_scheduling=self.async_scheduling,
            stream_interval=self.stream_interval,
        )

        if not model_config.is_multimodal_model and self.default_mm_loras:
            raise ValueError(
                "Default modality-specific LoRA(s) were provided for a "
                "non multimodal model"
            )

        lora_config = (
            LoRAConfig(
                max_lora_rank=self.max_lora_rank,
                max_loras=self.max_loras,
                default_mm_loras=self.default_mm_loras,
                fully_sharded_loras=self.fully_sharded_loras,
                lora_dtype=self.lora_dtype,
                target_modules=self.lora_target_modules,
                enable_tower_connector_lora=self.enable_tower_connector_lora,
                specialize_active_lora=self.specialize_active_lora,
                enable_mixed_moe_lora_format=self.enable_mixed_moe_lora_format,
                enable_moe_shared_loras=self.enable_moe_shared_loras,
                max_lora_cls_labels=self.max_lora_cls_labels,
                max_cpu_loras=self.max_cpu_loras
                if self.max_cpu_loras and self.max_cpu_loras > 0
                else None,
            )
            if self.enable_lora
            else None
        )

        if (
            lora_config is not None
            and speculative_config is not None
            and scheduler_config.max_num_batched_tokens
            < (
                scheduler_config.max_num_seqs
                * (speculative_config.num_speculative_tokens + 1)
            )
        ):
            raise ValueError(
                "Consider increasing max_num_batched_tokens or "
                "decreasing num_speculative_tokens"
            )

        # Attention config overrides
        attention_config = copy.deepcopy(self.attention_config)
        if self.attention_backend is not None:
            if attention_config.backend is not None:
                raise ValueError(
                    "attention_backend and attention_config.backend "
                    "are mutually exclusive"
                )
            # Reuse the validator to handle "auto" and string-to-enum conversion
            attention_config.backend = AttentionConfig.validate_backend_before(
                self.attention_backend
            )

        # Batch-invariant mode requires deterministic attention behavior.
        # If no backend is explicitly requested, prefer Triton Attention.
        if (
            envs.VLLM_BATCH_INVARIANT
            and attention_config.backend is None
            and current_platform.is_xpu()
        ):
            attention_config.backend = AttentionBackendEnum.TRITON_ATTN
            logger.info(
                "VLLM_BATCH_INVARIANT is enabled and no attention backend was "
                "specified; defaulting to TRITON_ATTN."
            )

        # TurboQuant requires FlashAttention 2 — FA3 boundary layers assert
        # FlashAttentionImpl which fails with TurboQuantAttentionImpl.
        # UltraQuant subclasses that impl, so it inherits the same limit.
        if uses_packed_kv_backend and (
            attention_config.flash_attn_version is None
            or attention_config.flash_attn_version >= 3
        ):
            logger.warning(
                "%s is not yet compatible with FlashAttention >= 3. "
                "Overriding flash_attn_version to 2. To silence this "
                "warning, pass --attention-config.flash_attn_version=2",
                "TurboQuant"
                if resolved_cache_dtype.startswith("turboquant_")
                else "UltraQuant",
            )
            attention_config.flash_attn_version = 2

        # Mamba config overrides
        mamba_config = copy.deepcopy(self.mamba_config)
        # Convert string to enum if needed (CLI parsing returns a string)
        if isinstance(self.mamba_backend, str):
            mamba_config.backend = MambaBackendEnum[self.mamba_backend.upper()]
        else:
            mamba_config.backend = self.mamba_backend
        if self.mamba_ssu_algorithm is not None:
            mamba_config.ssu_algorithm = self.mamba_ssu_algorithm
        if self.enable_mamba_cache_stochastic_rounding:
            mamba_config.enable_stochastic_rounding = (
                self.enable_mamba_cache_stochastic_rounding
            )
        if self.mamba_cache_philox_rounds:
            mamba_config.stochastic_rounding_philox_rounds = (
                self.mamba_cache_philox_rounds
            )
        mamba_config.validate_ssu_algorithm()

        # Kernel config overrides
        kernel_config = copy.deepcopy(self.kernel_config)
        if self.enable_flashinfer_autotune is not None:
            if kernel_config.enable_flashinfer_autotune is not None:
                raise ValueError(
                    "enable_flashinfer_autotune and "
                    "kernel_config.enable_flashinfer_autotune "
                    "are mutually exclusive"
                )
            kernel_config.enable_flashinfer_autotune = self.enable_flashinfer_autotune
        if self.monokernel:
            kernel_config.monokernel = True
        if self.moe_backend != "auto":
            kernel_config.moe_backend = self.moe_backend
        if self.linear_backend != "auto":
            kernel_config.linear_backend = self.linear_backend
        if self.sparse_indexer_topk_backend != "auto":
            kernel_config.sparse_indexer_topk_backend = self.sparse_indexer_topk_backend

        # Transfer top-level ir_op_priority into KernelConfig.ir_op_priority
        for op_name, op_priority in asdict(self.ir_op_priority).items():
            # Empty means unset
            if not op_priority:
                continue

            # Priority cannot be set 2x for the same op
            if getattr(kernel_config.ir_op_priority, op_name):
                raise ValueError(
                    f"Op priority for {op_name} specified via both ir_op_priority "
                    f"and KernelConfig.ir_op_priority, only one allowed at a time."
                )

            # Set the attribute
            setattr(kernel_config.ir_op_priority, op_name, op_priority)

        load_config = self.create_load_config()

        # Pass reasoning_parser into StructuredOutputsConfig
        self.create_structured_outputs_config()

        observability_config = self.create_observability_config()

        # Compilation config overrides
        compilation_config = copy.deepcopy(self.compilation_config)
        if self.cudagraph_capture_sizes is not None:
            if compilation_config.cudagraph_capture_sizes is not None:
                raise ValueError(
                    "cudagraph_capture_sizes and compilation_config."
                    "cudagraph_capture_sizes are mutually exclusive"
                )
            compilation_config.cudagraph_capture_sizes = self.cudagraph_capture_sizes
        if self.max_cudagraph_capture_size is not None:
            if compilation_config.max_cudagraph_capture_size is not None:
                raise ValueError(
                    "max_cudagraph_capture_size and compilation_config."
                    "max_cudagraph_capture_size are mutually exclusive"
                )
            compilation_config.max_cudagraph_capture_size = (
                self.max_cudagraph_capture_size
            )

        offload_config = OffloadConfig(
            offload_backend=self.offload_backend,
            uva=UVAOffloadConfig(
                cpu_offload_gb=self.cpu_offload_gb,
                cpu_offload_params=self.cpu_offload_params,
            ),
            prefetch=PrefetchOffloadConfig(
                offload_group_size=self.offload_group_size,
                offload_num_in_group=self.offload_num_in_group,
                offload_prefetch_step=self.offload_prefetch_step,
                offload_params=self.offload_params,
            ),
        )

        if self.gdn_prefill_backend is not None:
            self.additional_config["gdn_prefill_backend"] = self.gdn_prefill_backend
        if self.kda_prefill_backend is not None:
            if (
                self.kda_prefill_backend == "flashkda"
                and not current_platform.is_cuda()
            ):
                raise ValueError(
                    "--kda-prefill-backend=flashkda is only available on CUDA."
                )
            if self.kda_prefill_backend == "fused" and not current_platform.is_rocm():
                raise ValueError(
                    "--kda-prefill-backend=fused is only available on ROCm."
                )
            self.additional_config["kda_prefill_backend"] = self.kda_prefill_backend
        if self.kda_decode_backend is not None:
            self.additional_config["kda_decode_backend"] = self.kda_decode_backend

        config = VllmConfig(
            model_config=model_config,
            cache_config=cache_config,
            parallel_config=parallel_config,
            scheduler_config=scheduler_config,
            device_config=device_config,
            load_config=load_config,
            offload_config=offload_config,
            attention_config=attention_config,
            aux_output_config=self.aux_output_config,
            engram_config=self.engram_config,
            mamba_config=mamba_config,
            kernel_config=kernel_config,
            lora_config=lora_config,
            speculative_config=speculative_config,
            watermark_config=watermark_config,
            diffusion_config=diffusion_config,
            structured_outputs_config=self.structured_outputs_config,
            observability_config=observability_config,
            logging_config=logging_config,
            compilation_config=compilation_config,
            kv_transfer_config=self.kv_transfer_config,
            kv_events_config=self.kv_events_config,
            ec_transfer_config=self.ec_transfer_config,
            ec_manager_config=self.ec_manager_config,
            reasoning_config=self.reasoning_config,
            profiler_config=self.profiler_config,
            additional_config=self.additional_config,
            optimization_level=self.optimization_level,
            performance_mode=self.performance_mode,
            weight_transfer_config=self.weight_transfer_config,
            shutdown_timeout=self.shutdown_timeout,
        )

        return config

    def _check_feature_supported(self):
        """Raise an error if the feature is not supported."""
        if self.pipeline_parallel_size > 1:
            supports_pp = getattr(
                self.distributed_executor_backend, "supports_pp", False
            )
            if not supports_pp and self.distributed_executor_backend not in (
                ParallelConfig.distributed_executor_backend,
                "ray",
                "mp",
                "external_launcher",
            ):
                name = (
                    "Pipeline Parallelism without Ray distributed "
                    "executor or multiprocessing executor or external "
                    "launcher"
                )
                _raise_unsupported_error(feature_name=name)

    @classmethod
    def get_batch_defaults(
        cls,
        world_size: int,
    ) -> tuple[dict[UsageContext | None, int], dict[UsageContext | None, int]]:
        from vllm.usage.usage_lib import UsageContext

        default_max_num_batched_tokens: dict[UsageContext | None, int]
        default_max_num_seqs: dict[UsageContext | None, int]

        # When no user override, set the default values based on the usage
        # context.
        # Use different default values for different hardware.

        # Try to query the device name on the current platform. If it fails,
        # it may be because the platform that imports vLLM is not the same
        # as the platform that vLLM is running on (e.g. the case of scaling
        # vLLM with Ray) and has no GPUs. In this case we use the default
        # values for non-H100/H200 GPUs.
        try:
            device_memory = current_platform.get_device_total_memory()
            device_name = current_platform.get_device_name().lower()
            # GB10 (DGX Spark, cc 12.1) has the memory but not the compute of
            # a large discrete GPU, keep it on the small tier.
            is_gb10 = current_platform.is_device_capability((12, 1))
        except Exception:
            # This is only used to set default_max_num_batched_tokens
            device_memory = 0
            device_name = ""
            is_gb10 = False

        # NOTE(Kuntai): Setting large `max_num_batched_tokens` for A100 reduces
        # throughput, see PR #17885 for more details.
        # So here we do an extra device name check to prevent such regression.
        if device_memory >= 160 * GiB_bytes:
            # for GPUs like B200/B300 with >= 160GB memory, use the largest defaults
            default_max_num_batched_tokens = {
                UsageContext.LLM_CLASS: 16384,
                UsageContext.OPENAI_API_SERVER: 16384,
            }
            default_max_num_seqs = {
                UsageContext.LLM_CLASS: 1024,
                UsageContext.OPENAI_API_SERVER: 1024,
            }
        elif (
            device_memory >= 70 * GiB_bytes
            and "a100" not in device_name
            and not is_gb10
        ):
            # For GPUs like H100 and H200, use larger offline defaults.
            default_max_num_batched_tokens = {
                UsageContext.LLM_CLASS: 16384,
                UsageContext.OPENAI_API_SERVER: 8192,
            }
            default_max_num_seqs = {
                UsageContext.LLM_CLASS: 1024,
                UsageContext.OPENAI_API_SERVER: 1024,
            }
        else:
            # TODO(woosuk): Tune the default values for other hardware.
            default_max_num_batched_tokens = {
                UsageContext.LLM_CLASS: 8192,
                UsageContext.OPENAI_API_SERVER: 2048,
            }
            default_max_num_seqs = {
                UsageContext.LLM_CLASS: 256,
                UsageContext.OPENAI_API_SERVER: 256,
            }

        # tpu specific default values.
        if current_platform.is_tpu():
            chip_name = current_platform.get_device_name()

            if chip_name == "V6E":
                default_max_num_batched_tokens = {
                    UsageContext.LLM_CLASS: 2048,
                    UsageContext.OPENAI_API_SERVER: 1024,
                }
            elif chip_name == "V5E":
                default_max_num_batched_tokens = {
                    UsageContext.LLM_CLASS: 1024,
                    UsageContext.OPENAI_API_SERVER: 512,
                }
            elif chip_name == "V5P":
                default_max_num_batched_tokens = {
                    UsageContext.LLM_CLASS: 512,
                    UsageContext.OPENAI_API_SERVER: 256,
                }

        # cpu specific default values.
        if current_platform.is_cpu():
            default_max_num_batched_tokens = {
                UsageContext.LLM_CLASS: 4096 * world_size,
                UsageContext.OPENAI_API_SERVER: 2048 * world_size,
            }
            default_max_num_seqs = {
                UsageContext.LLM_CLASS: 256 * world_size,
                UsageContext.OPENAI_API_SERVER: 128 * world_size,
            }

        return default_max_num_batched_tokens, default_max_num_seqs

    def _set_default_chunked_prefill_and_prefix_caching_args(
        self, model_config: ModelConfig
    ) -> None:
        default_chunked_prefill = model_config.is_chunked_prefill_supported
        default_prefix_caching = model_config.is_prefix_caching_supported

        if self.enable_chunked_prefill is None:
            self.enable_chunked_prefill = default_chunked_prefill

            logger.debug(
                "%s chunked prefill by default",
                "Enabling" if default_chunked_prefill else "Disabling",
            )
        elif (
            model_config.runner_type == "generate"
            and not self.enable_chunked_prefill
            and default_chunked_prefill
        ):
            logger.warning_once(
                "This model does not officially support disabling chunked prefill. "
                "Disabling this manually may cause the engine to crash "
                "or produce incorrect outputs.",
            )
        elif (
            model_config.runner_type == "pooling"
            and self.enable_chunked_prefill
            and not default_chunked_prefill
        ):
            logger.warning_once(
                "This model does not officially support chunked prefill. "
                "Enabling this manually may cause the engine to crash "
                "or produce incorrect outputs.",
            )

        if self.enable_prefix_caching is None:
            self.enable_prefix_caching = default_prefix_caching

            logger.debug(
                "%s prefix caching by default",
                "Enabling" if default_prefix_caching else "Disabling",
            )
        elif (
            model_config.runner_type == "pooling"
            and self.enable_prefix_caching
            and not default_prefix_caching
        ):
            logger.warning_once(
                "This model does not officially support prefix caching. "
                "Enabling this manually may cause the engine to crash "
                "or produce incorrect outputs.",
            )

        # Disable chunked prefill and prefix caching for:
        # RISCV CPUs in V1
        if current_platform.is_cpu() and current_platform.get_cpu_architecture() in (
            CpuArchEnum.RISCV,
        ):
            logger.info(
                "Chunked prefill is not supported for"
                "RISC-V CPUs; "
                "disabling it for V1 backend."
            )
            self.enable_chunked_prefill = False
            logger.info(
                "Prefix caching is not supported for "
                "RISC-V CPUs; "
                "disabling it for V1 backend."
            )
            self.enable_prefix_caching = False

    def _set_default_reasoning_config_args(self):
        if not self.reasoning_parser:
            return
        if self.reasoning_config is None:
            self.reasoning_config = ReasoningConfig()
        self.reasoning_config.reasoning_parser = self.reasoning_parser

    @staticmethod
    def _get_min_mm_batched_tokens(
        model_config: ModelConfig,
    ) -> tuple[int, str] | None:
        """Get the minimum max_num_batched_tokens needed for a multimodal
        prefix-LM model to process at least one item of any supported modality.

        Returns (token_count, modality_name) for the most expensive modality,
        or None if the value cannot be determined at this stage.
        """
        try:
            from vllm.multimodal import MULTIMODAL_REGISTRY

            # get_processing_info returns the model's multimodal processing
            # metadata (supported modalities, token limits) without loading
            # model weights or generating dummy data.
            info = MULTIMODAL_REGISTRY.get_processing_info(model_config)
            mm_counts = {modality: 1 for modality in info.supported_mm_limits}
            # get_mm_max_tokens_per_item returns pre-computed per-item token
            # ceilings for models that override it (e.g., Gemma4), or None
            # for models that rely on dummy-input profiling. When None is
            # returned we bail out — no dummy generation is triggered here.
            max_tokens = info.get_mm_max_tokens_per_item(
                seq_len=model_config.max_model_len,
                mm_counts=mm_counts,
            )
            if max_tokens is not None:
                modality = max(max_tokens, key=max_tokens.__getitem__)
                return (max_tokens[modality], modality)
        except Exception as e:
            logger.warning("Failed to determine min multimodal batched tokens: %s", e)
        return None

    def _set_default_max_num_seqs_and_batched_tokens_args(
        self,
        usage_context: UsageContext | None,
        model_config: ModelConfig,
        parallel_config: ParallelConfig,
    ):
        world_size = self.pipeline_parallel_size * self.tensor_parallel_size
        (
            default_max_num_batched_tokens,
            default_max_num_seqs,
        ) = self.get_batch_defaults(world_size)

        orig_max_num_batched_tokens = self.max_num_batched_tokens
        orig_max_num_seqs = self.max_num_seqs

        if self.max_num_batched_tokens is None:
            if parallel_config.use_batched_dp_moe:
                self.max_num_batched_tokens = (
                    SchedulerConfig.DEFAULT_MAX_NUM_BATCHED_TOKENS_FOR_BATCHED_DP
                )
            else:
                self.max_num_batched_tokens = default_max_num_batched_tokens.get(
                    usage_context,
                    SchedulerConfig.DEFAULT_MAX_NUM_BATCHED_TOKENS,
                )

        if self.max_num_seqs is None:
            self.max_num_seqs = default_max_num_seqs.get(
                usage_context,
                SchedulerConfig.DEFAULT_MAX_NUM_SEQS,
            )

        # If throughput mode is set, double max_num_batched_tokens and max_num_seqs.
        if self.performance_mode == "throughput":
            if orig_max_num_batched_tokens is None:
                self.max_num_batched_tokens *= 2
            if orig_max_num_seqs is None:
                self.max_num_seqs *= 2

        if orig_max_num_batched_tokens is None:
            assert model_config.max_model_len is not None, (
                "max_model_len must be set by this point"
            )
            if not self.enable_chunked_prefill:
                # If max_model_len is too short, use the default for higher throughput.
                self.max_num_batched_tokens = max(
                    model_config.max_model_len,
                    self.max_num_batched_tokens,
                )

            # For multimodal prefix-LM models (e.g., Gemma 4) that disable
            # chunked MM input, a single multimodal item must fit in one batch.
            # Raise the floor to accommodate the largest per-item token count.
            if model_config.is_multimodal_model and model_config.is_mm_prefix_lm:
                result = self._get_min_mm_batched_tokens(model_config)
                if result is not None and result[0] > self.max_num_batched_tokens:
                    mm_min, modality = result
                    logger.info(
                        "Raising max_num_batched_tokens from %d to %d to "
                        "accommodate '%s' input for prefix-LM model %s.",
                        self.max_num_batched_tokens,
                        mm_min,
                        modality,
                        model_config.model,
                    )
                    self.max_num_batched_tokens = mm_min

            # When using default settings,
            # Ensure max_num_batched_tokens does not exceed model limit.
            # Some models (e.g., Whisper) have embeddings tied to max length.
            self.max_num_batched_tokens = min(
                self.max_num_seqs * model_config.max_model_len,
                self.max_num_batched_tokens,
            )

            logger.debug(
                "Defaulting max_num_batched_tokens to %d for %s usage context.",
                self.max_num_batched_tokens,
                usage_context.value if usage_context else None,
            )

        if orig_max_num_seqs is None:
            assert self.max_num_batched_tokens is not None  # For type checking
            self.max_num_seqs = min(self.max_num_seqs, self.max_num_batched_tokens)

            logger.debug(
                "Defaulting max_num_seqs to %d for %s usage context.",
                self.max_num_seqs,
                usage_context.value if usage_context else None,
            )


@dataclass
class AsyncEngineArgs(EngineArgs):
    """Arguments for asynchronous vLLM engine."""

    enable_log_requests: bool = False

    @staticmethod
    def add_cli_args(
        parser: FlexibleArgumentParser, async_args_only: bool = False
    ) -> FlexibleArgumentParser:
        # Initialize plugin to update the parser, for example, The plugin may
        # add a new kind of quantization method to --quantization argument or
        # a new device to --device argument.
        load_general_plugins()
        if not async_args_only:
            parser = EngineArgs.add_cli_args(parser)
        parser.add_argument(
            "--enable-log-requests",
            action=argparse.BooleanOptionalAction,
            default=AsyncEngineArgs.enable_log_requests,
            help="Enable logging request information, dependent on log level:\n"
            "- INFO: Request ID, parameters and LoRA request.\n"
            "- DEBUG: Prompt inputs (e.g: text, token IDs).\n"
            "You can set the minimum log level via `VLLM_LOGGING_LEVEL`.\n"
            "You can also set a separate log level for request JSON body "
            "via `VLLM_REQUEST_BODY_LOG_LEVEL` (default: DEBUG).",
        )
        current_platform.pre_register_and_update(parser)
        return parser


def _raise_unsupported_error(feature_name: str):
    msg = (
        f"{feature_name} is not supported. We recommend to "
        f"remove {feature_name} from your config."
    )
    raise NotImplementedError(msg)