[Mini-VLLM] Cut non-OpenAI surfaces, dead kernels, Mistral stack

- Remove Anthropic, Cohere, SageMaker, decisions, structured-decisions,
  rl, render-launcher and batch entrypoints; keep scale-out token routes
- Drop the Swagger static bundle and the /docs, /redoc routes
- Delete device-keyed tuning configs for fused_moe, quant utils, mamba
  select-state and helion; no sm86 entry ever existed for any of them
- Remove ten unused rope variants plus the fbgemm_fp8 and fp_quant
  quantizers and their config, metrics and loader plumbing
- Cut the CPU device build: csrc/cpu, cpu_extension.cmake, cpu
  requirements and the setup.py cpu target path
- Delete 29 environment variables nothing reads and 14 chat-template
  fallbacks for models this fork no longer registers
- Remove nine dead CUDA ops (deepseek_v4, kimi_k3, minimax, muse_spark,
  dsv3 gemm) with their kernels, bindings, wrappers and CMake entries
- Delete the connector test suites that targeted removed connectors
- Cut the Mistral tokenizer and parser stack, its tests, and the
  mistral_common and ijson dependency pins
- Restore the Qwen4Exp registry rows and nvidia/mtp.py that an earlier
  cut deleted while keeping the model tree

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 17:10:56 +07:00
co-authored by Hinata
parent bd70a61c20
commit 2066d3143f
1086 changed files with 122 additions and 291919 deletions
-151
View File
@@ -49,15 +49,10 @@ if TYPE_CHECKING:
VLLM_TRACE_FUNCTION: int = 0
VLLM_USE_FLASHINFER_SAMPLER: bool = True
VLLM_PP_LAYER_PARTITION: str | None = None
VLLM_CPU_KVCACHE_SPACE: int | None = 0
VLLM_CPU_OMP_THREADS_BIND: str = "auto"
VLLM_CPU_NUM_OF_RESERVED_CPU: int | None = None
VLLM_CPU_CI_ENV: bool = False
VLLM_CPU_ATTN_SPLIT_KV: bool = True
VLLM_ZENTORCH_WEIGHT_PREPACK: bool = True
VLLM_CPU_INT4_W4A8: bool = True
VLLM_XLA_CACHE_PATH: str = os.path.join(VLLM_CACHE_ROOT, "xla_cache")
VLLM_XLA_CHECK_RECOMPILATION: bool = False
VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: int = 512
VLLM_ADAPTIVE_VERIFICATION_PROFILE_CONTEXT_LEN: int = 8192
VLLM_XPU_PP_MICROBATCH: bool = True
@@ -97,10 +92,6 @@ if TYPE_CHECKING:
VLLM_USE_PRECOMPILED: bool = False
VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX: bool = False
VLLM_DOCKER_BUILD_CONTEXT: bool = False
VLLM_BUILD_COMMIT: str = "unknown"
VLLM_BUILD_PIPELINE: str = "local"
VLLM_BUILD_URL: str = ""
VLLM_IMAGE_TAG: str = ""
VLLM_KEEP_ALIVE_ON_ENGINE_DEATH: bool = False
CMAKE_BUILD_TYPE: Literal["Debug", "Release", "RelWithDebInfo"] | None = None
VERBOSE: bool = False
@@ -125,7 +116,6 @@ if TYPE_CHECKING:
VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False
VLLM_SKIP_P2P_CHECK: bool = False
VLLM_DISABLED_KERNELS: list[str] = []
VLLM_USE_HW_AGNOSTIC: bool = False
VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE: bool = True
VLLM_GDN_DECODE_KERNEL: Literal["cuda", "triton"] = "cuda"
VLLM_DISABLE_PYNCCL: bool = False
@@ -142,8 +132,6 @@ if TYPE_CHECKING:
VLLM_ROCM_USE_AITER_MOE_SITUV2: Literal["auto", "a4w4", "a8w4", "a16w4"] = "auto"
VLLM_ROCM_USE_AITER_RMSNORM: bool = True
VLLM_ROCM_USE_AITER_MLA: bool = True
VLLM_ROCM_AITER_MLA_ASM_PADDING: Literal["auto", "gluon", "asm"] = "auto"
VLLM_ROCM_AITER_MLA_DCP_VERIFY: Literal["auto", "asm", "segmented"] = "auto"
VLLM_ROCM_USE_AITER_MHA: bool = True
VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA: bool = False
VLLM_ROCM_USE_AITER_TRITON_ROPE: bool = False
@@ -160,12 +148,10 @@ if TYPE_CHECKING:
VLLM_ENABLE_V1_MULTIPROCESSING: bool = True
VLLM_LOG_BATCHSIZE_INTERVAL: float = -1
VLLM_DISABLE_COMPILE_CACHE: bool = False
VLLM_REPLICATE_EMBED: bool = False
VLLM_USE_LAYERNAME: bool = True
VLLM_SERVER_DEV_MODE: bool = False
VLLM_V1_OUTPUT_PROC_CHUNK_SIZE: int = 128
VLLM_MLA_DISABLE: bool = False
VLLM_USE_RAY_V2_EXECUTOR_BACKEND: bool = True
VLLM_RAY_PER_WORKER_GPUS: float = 1.0
VLLM_RAY_BUNDLE_INDICES: str = ""
VLLM_CUDART_SO_PATH: str | None = None
@@ -191,12 +177,10 @@ if TYPE_CHECKING:
VLLM_B12X_MOE_FP4_FORCE_A16: bool = False
VLLM_DEEPEPLL_NVFP4_DISPATCH: bool = False
VLLM_V1_USE_OUTLINES_CACHE: bool = False
VLLM_TPU_USING_PATHWAYS: bool = False
VLLM_USE_DEEP_GEMM: bool = True
VLLM_MOE_USE_DEEP_GEMM: bool = True
VLLM_USE_DEEP_GEMM_E8M0: bool = True
VLLM_USE_DEEP_GEMM_TMA_ALIGNED_SCALES: bool = True
VLLM_DCP_Q_REPLICATE: bool = False
VLLM_USE_DIRECT_DCP_A2A: bool | None = None
VLLM_USE_DIRECT_DCP_Q_GATHER: bool | None = None
VLLM_USE_DIRECT_DCP_KV_GATHER: bool | None = None
@@ -207,10 +191,6 @@ if TYPE_CHECKING:
] = "relax"
VLLM_USE_FUSED_MOE_GROUPED_TOPK: bool = True
VLLM_MOE_SKIP_PADDING: bool = True
VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: bool = False
VLLM_KIMI_K3_AUX_ATTN_RES_STREAM: bool = False
VLLM_KIMI_K3_GEMM_AR: bool = True
VLLM_ENABLE_GEMM_RS: bool = False
VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER: bool = True
VLLM_USE_FLASHINFER_MOE_INT4: bool = False
VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR: str | None = None
@@ -224,25 +204,15 @@ if TYPE_CHECKING:
VLLM_ALLOW_INSECURE_SERIALIZATION: bool = False
VLLM_ABLITERATION: str = ""
VLLM_DISABLE_REQUEST_ID_RANDOMIZATION: bool = False
VLLM_NIXL_SIDE_CHANNEL_HOST: str = "localhost"
VLLM_NIXL_SIDE_CHANNEL_PORT: int = 5600
VLLM_P2P_SIDE_CHANNEL_HOST: str = "localhost"
VLLM_P2P_SIDE_CHANNEL_PORT: int = 5710
VLLM_P2P_MAX_PEERS: int = 128
VLLM_P2P_HANDSHAKE_TIMEOUT_S: int = 30
VLLM_P2P_IDLE_TIMEOUT_S: int = 300
VLLM_EC_SIDE_CHANNEL_HOST: str = "localhost"
VLLM_EC_SIDE_CHANNEL_PORT: int = 5601
VLLM_MOONCAKE_BOOTSTRAP_PORT: int = 8998
VLLM_MOONCAKE_STORE_TIER_LOG: bool = False
VLLM_MOONCAKE_LOAD_RECV_THREADS: int = 1
VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO: float = 0.9
MOONCAKE_PREFERRED_SEGMENT: str | None = None
MOONCAKE_REQUESTER_LOCAL_HOSTNAME: str | None = None
VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: int = 163840
VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS: int = 1
VLLM_ENFORCE_STRICT_TOOL_CALLING: bool = True
VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT: bool = True
VLLM_MQ_MAX_CHUNK_BYTES_MB: int = 16
VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: int = 300
VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS: int = 5
@@ -260,13 +230,10 @@ if TYPE_CHECKING:
VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB: int | None = None
VLLM_ROCM_QUICK_REDUCE_MIN_SIZE_BYTES_MB: int | None = None
VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB: int | None = None
VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT: int = 480
VLLM_MOONCAKE_CONNECTOR_TIMEOUT: float = 30.0
VLLM_ENABLE_CUDAGRAPH_GC: bool = False
VLLM_LOOPBACK_IP: str = ""
VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE: bool = True
VLLM_ENABLE_RESPONSES_API_STORE: bool = False
VLLM_ENABLE_COHERE_API: bool = False
VLLM_HAS_FLASHINFER_CUBIN: bool = False
VLLM_ROCM_FP8_MFMA_PAGE_ATTN: bool = False
VLLM_ALLREDUCE_USE_SYMM_MEM: bool = True
@@ -299,9 +266,7 @@ if TYPE_CHECKING:
VLLM_GC_DEBUG: str = ""
VLLM_DEBUG_WORKSPACE: bool = False
VLLM_DISABLE_SHARED_EXPERTS_STREAM: bool = False
VLLM_DISABLE_DSV4_MEGAMOE_SHARED_EXPERT_FUSION: bool = False
VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD: int = 256
VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD: int = 1024
VLLM_COMPILE_CACHE_SAVE_FORMAT: Literal["binary", "unpacked"] = "binary"
VLLM_USE_V2_MODEL_RUNNER: bool | None = None
VLLM_LOG_MODEL_INSPECTION: bool = False
@@ -323,11 +288,9 @@ if TYPE_CHECKING:
VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: bool = True
VLLM_XPU_FORCE_N_CONTIG_WEIGHT: bool = False
VLLM_XPU_USE_SAMPLER_KERNEL: bool = True
VLLM_XPU_INC_WNA16_BACKEND: Literal["auto", "ark", "w4a16", "w4a8"] = "auto"
VLLM_LORA_ENABLE_DUAL_STREAM: bool = False
VLLM_GPU_NIC_PCIE_MAPPING: str = ""
VLLM_NIC_SELECTION_VARS: str = ""
VLLM_ENABLE_HPC_OPS: bool = False
VLLM_REQUEST_BODY_LOG_LEVEL: str = "DEBUG"
@@ -595,9 +558,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
# Enable batch-invariant mode: deterministic results regardless of
# batch composition. Requires NVIDIA GPU with compute capability >= 9.0.
"VLLM_BATCH_INVARIANT": lambda: bool(int(os.getenv("VLLM_BATCH_INVARIANT", "0"))),
"VLLM_REPLICATE_EMBED": lambda: (
os.getenv("VLLM_REPLICATE_EMBED", "0").strip().lower() in ("1", "true")
),
# Use tensor descriptors for Q/K/V loads and output stores in the
# Triton unified-attention kernel. Enables HW 2D block reads on
# Intel XPU; the non-TD branch is dead-code-eliminated at Triton
@@ -640,10 +600,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
),
# Build provenance metadata embedded in official vllm-openai images.
# Set via Docker ENV at image build time; informational only.
"VLLM_BUILD_COMMIT": lambda: os.environ.get("VLLM_BUILD_COMMIT", "unknown"),
"VLLM_BUILD_PIPELINE": lambda: os.environ.get("VLLM_BUILD_PIPELINE", "local"),
"VLLM_BUILD_URL": lambda: os.environ.get("VLLM_BUILD_URL", ""),
"VLLM_IMAGE_TAG": lambda: os.environ.get("VLLM_IMAGE_TAG", ""),
# CMake build type
# If not set, defaults to "Debug" or "RelWithDebInfo"
# Available options: "Debug", "Release", "RelWithDebInfo"
@@ -839,13 +795,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
),
# Pipeline stage partition strategy
"VLLM_PP_LAYER_PARTITION": lambda: os.getenv("VLLM_PP_LAYER_PARTITION", None),
# (CPU backend only) CPU key-value cache space.
# default is None and will be set as 4 GB
"VLLM_CPU_KVCACHE_SPACE": lambda: (
int(os.getenv("VLLM_CPU_KVCACHE_SPACE", "0"))
if "VLLM_CPU_KVCACHE_SPACE" in os.environ
else None
),
# (CPU backend only) CPU core ids bound by OpenMP threads, e.g., "0-31",
# "0,1,2", "0-31,33". CPU cores of different ranks are separated by '|'.
"VLLM_CPU_OMP_THREADS_BIND": lambda: os.getenv("VLLM_CPU_OMP_THREADS_BIND", "auto"),
@@ -856,12 +805,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
if "VLLM_CPU_NUM_OF_RESERVED_CPU" in os.environ
else None
),
# (CPU backend only) whether vLLM is running in a CI environment.
"VLLM_CPU_CI_ENV": lambda: bool(int(os.getenv("VLLM_CPU_CI_ENV", "0"))),
# (CPU backend only) whether to enable attention spilt KV.
"VLLM_CPU_ATTN_SPLIT_KV": lambda: bool(
int(os.getenv("VLLM_CPU_ATTN_SPLIT_KV", "1"))
),
# (Zen CPU backend) eagerly prepack weights into ZenDNN blocked layout
# at model load time. Eliminates per-inference layout conversion overhead.
"VLLM_ZENTORCH_WEIGHT_PREPACK": lambda: bool(
@@ -1025,16 +968,7 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_MEDIA_CONNECTOR": lambda: os.getenv("VLLM_MEDIA_CONNECTOR", "http"),
# Path to the XLA persistent cache directory.
# Only used for XLA devices such as TPUs.
"VLLM_XLA_CACHE_PATH": lambda: os.path.expanduser(
os.getenv(
"VLLM_XLA_CACHE_PATH",
os.path.join(get_default_cache_root(), "vllm", "xla_cache"),
)
),
# If set, assert on XLA recompilation after each execution step.
"VLLM_XLA_CHECK_RECOMPILATION": lambda: bool(
int(os.getenv("VLLM_XLA_CHECK_RECOMPILATION", "0"))
),
# Enable SPMD mode for TPU backend.
"VLLM_XLA_USE_SPMD": lambda: bool(int(os.getenv("VLLM_XLA_USE_SPMD", "0"))),
# Maximum size (in MB) for logits tensor in sparse MLA indexer prefill chunks.
@@ -1173,9 +1107,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
else os.environ["VLLM_DISABLED_KERNELS"].split(",")
),
# Selects hw-agnostic layers for HF transformer backend
"VLLM_USE_HW_AGNOSTIC": lambda: (
os.getenv("VLLM_USE_HW_AGNOSTIC", "False").lower() in ("true", "1")
),
"VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE": lambda: bool(
int(os.getenv("VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE", "1"))
),
@@ -1277,12 +1208,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
# context parallelism: "asm" uses AITER's round-robin ASM decode (gfx950),
# "segmented" the Triton segmented MLA path. "auto" (default) takes "asm"
# wherever it can serve the shape and "segmented" otherwise.
"VLLM_ROCM_AITER_MLA_DCP_VERIFY": env_with_choices(
"VLLM_ROCM_AITER_MLA_DCP_VERIFY",
"auto",
["auto", "asm", "segmented"],
case_sensitive=False,
),
# Small-head (<16) AITER MLA decode kernel selection. Small head counts
# (e.g. Kimi-K3: 12 heads/rank at TP8, 6 at TP16) can decode either through
# the Gluon small-head kernel or through the padded persistent-scheduling
@@ -1291,12 +1216,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
# PS ASM decode; "gluon" forces the Gluon path wherever a build exists;
# "asm" forces the padded PS ASM decode. On gfx942/CDNA3 there is no Gluon
# build, so the ASM path is always used regardless of this setting.
"VLLM_ROCM_AITER_MLA_ASM_PADDING": env_with_choices(
"VLLM_ROCM_AITER_MLA_ASM_PADDING",
"auto",
["auto", "gluon", "asm"],
case_sensitive=False,
),
# Whether to use aiter mha ops.
# By default is enabled.
"VLLM_ROCM_USE_AITER_MHA": lambda: (
@@ -1439,9 +1358,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
# Deprecated and ignored: RayExecutorV2 is the only Ray executor. Kept so
# out-of-tree platforms that still read it keep working; will be removed
# in v0.34.
"VLLM_USE_RAY_V2_EXECUTOR_BACKEND": lambda: bool(
int(os.getenv("VLLM_USE_RAY_V2_EXECUTOR_BACKEND", "1"))
),
# In some system, find_loaded_library() may not work. So we allow users to
# specify the path through environment variable VLLM_CUDART_SO_PATH.
"VLLM_CUDART_SO_PATH": lambda: os.getenv("VLLM_CUDART_SO_PATH", None),
@@ -1546,9 +1462,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
os.environ.get("VLLM_V1_USE_OUTLINES_CACHE", "0") == "1"
),
# Whether using Pathways
"VLLM_TPU_USING_PATHWAYS": lambda: bool(
"proxy" in os.getenv("JAX_PLATFORMS", "").lower()
),
# Allow use of DeepGemm kernels for fused moe ops.
"VLLM_USE_DEEP_GEMM": lambda: bool(int(os.getenv("VLLM_USE_DEEP_GEMM", "1"))),
# Allow use of DeepGemm specifically for MoE fused ops (overrides only MoE).
@@ -1564,7 +1477,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
int(os.getenv("VLLM_USE_DEEP_GEMM_TMA_ALIGNED_SCALES", "1"))
),
# Opt-in MLA DCP query replication: skip the decode query all-gather.
"VLLM_DCP_Q_REPLICATE": lambda: bool(int(os.getenv("VLLM_DCP_Q_REPLICATE", "0"))),
# DeepGemm JITs the kernels on-demand. The warmup attempts to make DeepGemm
# JIT all the required kernels before model execution so there is no
# JIT'ing in the hot-path. However, this warmup increases the engine
@@ -1602,24 +1514,16 @@ environment_variables: dict[str, Callable[[], Any]] = {
# per layer, so it only wins at low token counts: intended for decode
# instances in a P/D disaggregated deployment, not for prefill or unified
# serving.
"VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT": lambda: bool(
int(os.getenv("VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT", "0"))
),
# Kimi K3 only, and unrelated to the MoE flags above. Tap the pre-norm
# AttnRes mixture, rather than the post-mixture sum, as the auxiliary
# hidden state handed to a DFlash drafter. This changes the numerics the
# speculator sees, so it is off by default while the effect is measured.
"VLLM_KIMI_K3_AUX_ATTN_RES_STREAM": lambda: bool(
int(os.getenv("VLLM_KIMI_K3_AUX_ATTN_RES_STREAM", "0"))
),
# Use the SM100 BF16 GEMM-AR kernel for eligible Kimi-K3 row-parallel
# attention projections. All TP ranks must belong to one NVLink domain.
"VLLM_KIMI_K3_GEMM_AR": lambda: bool(int(os.getenv("VLLM_KIMI_K3_GEMM_AR", "1"))),
# Fuse eligible sequence-parallel row-parallel projections with their TP
# reduce-scatter using the SM100 BF16/MXFP8 GEMM-RS kernel (Kimi-K3
# attention/shared-expert projections, DeepSeek-V4.1 ``wo_b``). All TP
# ranks must belong to one NVLink domain.
"VLLM_ENABLE_GEMM_RS": lambda: bool(int(os.getenv("VLLM_ENABLE_GEMM_RS", "0"))),
# Allow use of FlashInfer FP8 block-scale GEMM for linear layers.
# This uses TensorRT-LLM kernels and requires SM90+ (Hopper).
"VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": lambda: bool(
@@ -1666,13 +1570,7 @@ environment_variables: dict[str, Callable[[], Any]] = {
int(os.getenv("VLLM_DISABLE_REQUEST_ID_RANDOMIZATION", "0"))
),
# IP address used for NIXL handshake between remote agents.
"VLLM_NIXL_SIDE_CHANNEL_HOST": lambda: os.getenv(
"VLLM_NIXL_SIDE_CHANNEL_HOST", "localhost"
),
# Port used for NIXL handshake between remote agents.
"VLLM_NIXL_SIDE_CHANNEL_PORT": lambda: int(
os.getenv("VLLM_NIXL_SIDE_CHANNEL_PORT", "5600")
),
# Address the P2P KV-offload control socket binds to. Defaults to
# ``localhost`` (loopback only); must be set to the node IP for
# cross-host P2P so remote peers can reach the socket.
@@ -1700,39 +1598,18 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_P2P_IDLE_TIMEOUT_S": lambda: int(os.getenv("VLLM_P2P_IDLE_TIMEOUT_S", "300")),
# IP address used for the EC connector's ZMQ side channel
# (producer ROUTER bind, consumer DEALER dial).
"VLLM_EC_SIDE_CHANNEL_HOST": lambda: os.getenv(
"VLLM_EC_SIDE_CHANNEL_HOST", "localhost"
),
# Port for the EC connector's ZMQ side channel; advertised to peers
# via `ec_transfer_params.peer_port` on the producer's response.
"VLLM_EC_SIDE_CHANNEL_PORT": lambda: int(
os.getenv("VLLM_EC_SIDE_CHANNEL_PORT", "5601")
),
# Per-request timeout (seconds) when a prefiller worker registers with
# the Mooncake bootstrap server.
"VLLM_MOONCAKE_CONNECTOR_TIMEOUT": lambda: float(
os.getenv("VLLM_MOONCAKE_CONNECTOR_TIMEOUT", "30.0")
),
# Port used for Mooncake handshake between remote agents.
"VLLM_MOONCAKE_BOOTSTRAP_PORT": lambda: int(
os.getenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", "8998")
),
# Log per-batch memory/disk tier breakdown on external GETs.
"VLLM_MOONCAKE_STORE_TIER_LOG": lambda: (
os.getenv("VLLM_MOONCAKE_STORE_TIER_LOG", "False").lower() in ("true", "1")
),
# Number of parallel KV-load receive threads per worker rank. Lets the
# per-request control overhead (Python prep + master key lookup) of one
# request overlap with the RDMA transfer of another, keeping the transfer
# engine's queue pairs busy. Helps when that overhead is significant or
# per-request batches are too small to saturate the link on their own.
"VLLM_MOONCAKE_LOAD_RECV_THREADS": lambda: int(
os.getenv("VLLM_MOONCAKE_LOAD_RECV_THREADS", "1")
),
# Fraction of the owner's DirectIO staging buffer to fill per GET batch.
"VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO": lambda: float(
os.getenv("VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO", "0.9")
),
# Pin this rank to a specific owner segment ("host:port").
"MOONCAKE_PREFERRED_SEGMENT": lambda: os.getenv("MOONCAKE_PREFERRED_SEGMENT"),
# Override the hostname the rank registers as a Mooncake requester.
@@ -1788,18 +1665,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_MOE_ROUTING_SIMULATION_STRATEGY", ""
).lower(),
# Regex timeout for use by the vLLM tool parsing plugins.
"VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS": lambda: int(
os.getenv("VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS", "1")
),
# Enforce function parameter schemas in structural-tag based tool calling.
"VLLM_ENFORCE_STRICT_TOOL_CALLING": lambda: (
os.getenv("VLLM_ENFORCE_STRICT_TOOL_CALLING", "True").lower() in ("true", "1")
),
# Constrain every `beam` output to its response template's format, also
# without tool or structured output constraints.
"VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT": lambda: (
os.getenv("VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT", "True").lower() in ("true", "1")
),
# Control the max chunk bytes (in MB) for the rpc message queue.
# Object larger than this threshold will be broadcast to worker
# processes via zmq.
@@ -1849,9 +1720,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
int(os.getenv("VLLM_RAISE_ON_LOGIT_NANS", "0"))
),
# Timeout (in seconds) for MooncakeConnector in PD disaggregated setup.
"VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT": lambda: int(
os.getenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "480")
),
# If set, it means we pre-downloaded cubin files and flashinfer will
# read the cubin files directly.
"VLLM_HAS_FLASHINFER_CUBIN": lambda: bool(
@@ -1891,12 +1759,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_ENABLE_RESPONSES_API_STORE": lambda: bool(
int(os.getenv("VLLM_ENABLE_RESPONSES_API_STORE", "0"))
),
# If set to 1, expose the Cohere Chat v2 API at ``POST /cohere/v2/chat``
# and its render endpoint at ``POST /cohere/v2/chat/render``.
# Default off
"VLLM_ENABLE_COHERE_API": lambda: bool(
int(os.getenv("VLLM_ENABLE_COHERE_API", "0"))
),
# If set, use the fp8 mfma in rocm paged attention.
"VLLM_ROCM_FP8_MFMA_PAGE_ATTN": lambda: bool(
int(os.getenv("VLLM_ROCM_FP8_MFMA_PAGE_ATTN", "0"))
@@ -2056,9 +1918,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
# Emergency rollback for the DeepSeek-V4 NVIDIA MegaMoE path. By default,
# DeepGEMM computes replicated FP8 shared experts in the same persistent
# SM100 kernel as the routed FP4 experts.
"VLLM_DISABLE_DSV4_MEGAMOE_SHARED_EXPERT_FUSION": lambda: bool(
int(os.getenv("VLLM_DISABLE_DSV4_MEGAMOE_SHARED_EXPERT_FUSION", "0"))
),
# Limits when we run shared_experts in a separate stream.
# We found out that for large batch sizes, the separate stream
# execution is not beneficial (most likely because of the input clone)
@@ -2074,9 +1933,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
# and the cross-stream sync becomes pure overhead. Set to 0 to disable
# the multi-stream path entirely. See #PR 41526 for the empirical result
# for the default value of 1024 tokens.
"VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD": lambda: int(
os.getenv("VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD", "1024")
),
# Format for saving torch.compile cache artifacts
# - "binary": saves as binary file
# Safe for multiple vllm serve processes accessing the same torch compile cache.
@@ -2175,9 +2031,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
# activations to per-token int8 (int4_gemm_w4a8) for large token counts.
# The two oneDNN backends are int4-only; ARK also serves int2.
# Which one is fastest is device-dependent, so this is left as an opt-in.
"VLLM_XPU_INC_WNA16_BACKEND": env_with_choices(
"VLLM_XPU_INC_WNA16_BACKEND", "auto", ["auto", "ark", "w4a16", "w4a8"]
),
# Direct DCP ops default on when applicable; set to 1 to enforce or 0 to disable.
"VLLM_USE_DIRECT_DCP_A2A": lambda: maybe_convert_bool(
os.getenv("VLLM_USE_DIRECT_DCP_A2A")
@@ -2212,7 +2065,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
# fused into gated_mla_gemm; elementwise gating only.
# Each op additionally checks its own shape / dtype constraints and falls
# back to the eager path when they do not hold.
"VLLM_ENABLE_HPC_OPS": lambda: bool(int(os.getenv("VLLM_ENABLE_HPC_OPS", "0"))),
# Whether to skip version suffix when building package
"VLLM_SKIP_VERSION_SUFFIX": lambda: bool(
int(os.getenv("VLLM_SKIP_VERSION_SUFFIX", "0"))
@@ -2308,13 +2160,11 @@ def compile_factors() -> dict[str, object]:
# HOME or the XDG roots silently invalidates every compile cache
# (VLLM_CACHE_ROOT above and VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR below
# are already ignored for the same reason).
"VLLM_XLA_CACHE_PATH",
"VLLM_CONFIG_ROOT",
"LD_LIBRARY_PATH",
"VLLM_SERVER_DEV_MODE",
"VLLM_DP_MASTER_IP",
"VLLM_DP_MASTER_PORT",
"VLLM_NIXL_SIDE_CHANNEL_HOST",
"VLLM_RANDOMIZE_DP_DUMMY_INPUTS",
"VLLM_MODEL_REDIRECT_PATH",
"VLLM_HOST_IP",
@@ -2369,7 +2219,6 @@ def compile_factors() -> dict[str, object]:
"VLLM_WORKER_MULTIPROC_METHOD",
"VLLM_ENABLE_V1_MULTIPROCESSING",
"VLLM_V1_OUTPUT_PROC_CHUNK_SIZE",
"VLLM_CPU_KVCACHE_SPACE",
"VLLM_CPU_MOE_PREPACK",
"VLLM_ZENTORCH_WEIGHT_PREPACK",
"VLLM_TEST_FORCE_LOAD_FORMAT",