[Mini-VLLM] Cut non-OpenAI surfaces, dead kernels, Mistral stack
- Remove Anthropic, Cohere, SageMaker, decisions, structured-decisions, rl, render-launcher and batch entrypoints; keep scale-out token routes - Drop the Swagger static bundle and the /docs, /redoc routes - Delete device-keyed tuning configs for fused_moe, quant utils, mamba select-state and helion; no sm86 entry ever existed for any of them - Remove ten unused rope variants plus the fbgemm_fp8 and fp_quant quantizers and their config, metrics and loader plumbing - Cut the CPU device build: csrc/cpu, cpu_extension.cmake, cpu requirements and the setup.py cpu target path - Delete 29 environment variables nothing reads and 14 chat-template fallbacks for models this fork no longer registers - Remove nine dead CUDA ops (deepseek_v4, kimi_k3, minimax, muse_spark, dsv3 gemm) with their kernels, bindings, wrappers and CMake entries - Delete the connector test suites that targeted removed connectors - Cut the Mistral tokenizer and parser stack, its tests, and the mistral_common and ijson dependency pins - Restore the Qwen4Exp registry rows and nvidia/mtp.py that an earlier cut deleted while keeping the model tree Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
-151
@@ -49,15 +49,10 @@ if TYPE_CHECKING:
|
||||
VLLM_TRACE_FUNCTION: int = 0
|
||||
VLLM_USE_FLASHINFER_SAMPLER: bool = True
|
||||
VLLM_PP_LAYER_PARTITION: str | None = None
|
||||
VLLM_CPU_KVCACHE_SPACE: int | None = 0
|
||||
VLLM_CPU_OMP_THREADS_BIND: str = "auto"
|
||||
VLLM_CPU_NUM_OF_RESERVED_CPU: int | None = None
|
||||
VLLM_CPU_CI_ENV: bool = False
|
||||
VLLM_CPU_ATTN_SPLIT_KV: bool = True
|
||||
VLLM_ZENTORCH_WEIGHT_PREPACK: bool = True
|
||||
VLLM_CPU_INT4_W4A8: bool = True
|
||||
VLLM_XLA_CACHE_PATH: str = os.path.join(VLLM_CACHE_ROOT, "xla_cache")
|
||||
VLLM_XLA_CHECK_RECOMPILATION: bool = False
|
||||
VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: int = 512
|
||||
VLLM_ADAPTIVE_VERIFICATION_PROFILE_CONTEXT_LEN: int = 8192
|
||||
VLLM_XPU_PP_MICROBATCH: bool = True
|
||||
@@ -97,10 +92,6 @@ if TYPE_CHECKING:
|
||||
VLLM_USE_PRECOMPILED: bool = False
|
||||
VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX: bool = False
|
||||
VLLM_DOCKER_BUILD_CONTEXT: bool = False
|
||||
VLLM_BUILD_COMMIT: str = "unknown"
|
||||
VLLM_BUILD_PIPELINE: str = "local"
|
||||
VLLM_BUILD_URL: str = ""
|
||||
VLLM_IMAGE_TAG: str = ""
|
||||
VLLM_KEEP_ALIVE_ON_ENGINE_DEATH: bool = False
|
||||
CMAKE_BUILD_TYPE: Literal["Debug", "Release", "RelWithDebInfo"] | None = None
|
||||
VERBOSE: bool = False
|
||||
@@ -125,7 +116,6 @@ if TYPE_CHECKING:
|
||||
VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False
|
||||
VLLM_SKIP_P2P_CHECK: bool = False
|
||||
VLLM_DISABLED_KERNELS: list[str] = []
|
||||
VLLM_USE_HW_AGNOSTIC: bool = False
|
||||
VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE: bool = True
|
||||
VLLM_GDN_DECODE_KERNEL: Literal["cuda", "triton"] = "cuda"
|
||||
VLLM_DISABLE_PYNCCL: bool = False
|
||||
@@ -142,8 +132,6 @@ if TYPE_CHECKING:
|
||||
VLLM_ROCM_USE_AITER_MOE_SITUV2: Literal["auto", "a4w4", "a8w4", "a16w4"] = "auto"
|
||||
VLLM_ROCM_USE_AITER_RMSNORM: bool = True
|
||||
VLLM_ROCM_USE_AITER_MLA: bool = True
|
||||
VLLM_ROCM_AITER_MLA_ASM_PADDING: Literal["auto", "gluon", "asm"] = "auto"
|
||||
VLLM_ROCM_AITER_MLA_DCP_VERIFY: Literal["auto", "asm", "segmented"] = "auto"
|
||||
VLLM_ROCM_USE_AITER_MHA: bool = True
|
||||
VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA: bool = False
|
||||
VLLM_ROCM_USE_AITER_TRITON_ROPE: bool = False
|
||||
@@ -160,12 +148,10 @@ if TYPE_CHECKING:
|
||||
VLLM_ENABLE_V1_MULTIPROCESSING: bool = True
|
||||
VLLM_LOG_BATCHSIZE_INTERVAL: float = -1
|
||||
VLLM_DISABLE_COMPILE_CACHE: bool = False
|
||||
VLLM_REPLICATE_EMBED: bool = False
|
||||
VLLM_USE_LAYERNAME: bool = True
|
||||
VLLM_SERVER_DEV_MODE: bool = False
|
||||
VLLM_V1_OUTPUT_PROC_CHUNK_SIZE: int = 128
|
||||
VLLM_MLA_DISABLE: bool = False
|
||||
VLLM_USE_RAY_V2_EXECUTOR_BACKEND: bool = True
|
||||
VLLM_RAY_PER_WORKER_GPUS: float = 1.0
|
||||
VLLM_RAY_BUNDLE_INDICES: str = ""
|
||||
VLLM_CUDART_SO_PATH: str | None = None
|
||||
@@ -191,12 +177,10 @@ if TYPE_CHECKING:
|
||||
VLLM_B12X_MOE_FP4_FORCE_A16: bool = False
|
||||
VLLM_DEEPEPLL_NVFP4_DISPATCH: bool = False
|
||||
VLLM_V1_USE_OUTLINES_CACHE: bool = False
|
||||
VLLM_TPU_USING_PATHWAYS: bool = False
|
||||
VLLM_USE_DEEP_GEMM: bool = True
|
||||
VLLM_MOE_USE_DEEP_GEMM: bool = True
|
||||
VLLM_USE_DEEP_GEMM_E8M0: bool = True
|
||||
VLLM_USE_DEEP_GEMM_TMA_ALIGNED_SCALES: bool = True
|
||||
VLLM_DCP_Q_REPLICATE: bool = False
|
||||
VLLM_USE_DIRECT_DCP_A2A: bool | None = None
|
||||
VLLM_USE_DIRECT_DCP_Q_GATHER: bool | None = None
|
||||
VLLM_USE_DIRECT_DCP_KV_GATHER: bool | None = None
|
||||
@@ -207,10 +191,6 @@ if TYPE_CHECKING:
|
||||
] = "relax"
|
||||
VLLM_USE_FUSED_MOE_GROUPED_TOPK: bool = True
|
||||
VLLM_MOE_SKIP_PADDING: bool = True
|
||||
VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: bool = False
|
||||
VLLM_KIMI_K3_AUX_ATTN_RES_STREAM: bool = False
|
||||
VLLM_KIMI_K3_GEMM_AR: bool = True
|
||||
VLLM_ENABLE_GEMM_RS: bool = False
|
||||
VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER: bool = True
|
||||
VLLM_USE_FLASHINFER_MOE_INT4: bool = False
|
||||
VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR: str | None = None
|
||||
@@ -224,25 +204,15 @@ if TYPE_CHECKING:
|
||||
VLLM_ALLOW_INSECURE_SERIALIZATION: bool = False
|
||||
VLLM_ABLITERATION: str = ""
|
||||
VLLM_DISABLE_REQUEST_ID_RANDOMIZATION: bool = False
|
||||
VLLM_NIXL_SIDE_CHANNEL_HOST: str = "localhost"
|
||||
VLLM_NIXL_SIDE_CHANNEL_PORT: int = 5600
|
||||
VLLM_P2P_SIDE_CHANNEL_HOST: str = "localhost"
|
||||
VLLM_P2P_SIDE_CHANNEL_PORT: int = 5710
|
||||
VLLM_P2P_MAX_PEERS: int = 128
|
||||
VLLM_P2P_HANDSHAKE_TIMEOUT_S: int = 30
|
||||
VLLM_P2P_IDLE_TIMEOUT_S: int = 300
|
||||
VLLM_EC_SIDE_CHANNEL_HOST: str = "localhost"
|
||||
VLLM_EC_SIDE_CHANNEL_PORT: int = 5601
|
||||
VLLM_MOONCAKE_BOOTSTRAP_PORT: int = 8998
|
||||
VLLM_MOONCAKE_STORE_TIER_LOG: bool = False
|
||||
VLLM_MOONCAKE_LOAD_RECV_THREADS: int = 1
|
||||
VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO: float = 0.9
|
||||
MOONCAKE_PREFERRED_SEGMENT: str | None = None
|
||||
MOONCAKE_REQUESTER_LOCAL_HOSTNAME: str | None = None
|
||||
VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: int = 163840
|
||||
VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS: int = 1
|
||||
VLLM_ENFORCE_STRICT_TOOL_CALLING: bool = True
|
||||
VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT: bool = True
|
||||
VLLM_MQ_MAX_CHUNK_BYTES_MB: int = 16
|
||||
VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: int = 300
|
||||
VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS: int = 5
|
||||
@@ -260,13 +230,10 @@ if TYPE_CHECKING:
|
||||
VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB: int | None = None
|
||||
VLLM_ROCM_QUICK_REDUCE_MIN_SIZE_BYTES_MB: int | None = None
|
||||
VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB: int | None = None
|
||||
VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT: int = 480
|
||||
VLLM_MOONCAKE_CONNECTOR_TIMEOUT: float = 30.0
|
||||
VLLM_ENABLE_CUDAGRAPH_GC: bool = False
|
||||
VLLM_LOOPBACK_IP: str = ""
|
||||
VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE: bool = True
|
||||
VLLM_ENABLE_RESPONSES_API_STORE: bool = False
|
||||
VLLM_ENABLE_COHERE_API: bool = False
|
||||
VLLM_HAS_FLASHINFER_CUBIN: bool = False
|
||||
VLLM_ROCM_FP8_MFMA_PAGE_ATTN: bool = False
|
||||
VLLM_ALLREDUCE_USE_SYMM_MEM: bool = True
|
||||
@@ -299,9 +266,7 @@ if TYPE_CHECKING:
|
||||
VLLM_GC_DEBUG: str = ""
|
||||
VLLM_DEBUG_WORKSPACE: bool = False
|
||||
VLLM_DISABLE_SHARED_EXPERTS_STREAM: bool = False
|
||||
VLLM_DISABLE_DSV4_MEGAMOE_SHARED_EXPERT_FUSION: bool = False
|
||||
VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD: int = 256
|
||||
VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD: int = 1024
|
||||
VLLM_COMPILE_CACHE_SAVE_FORMAT: Literal["binary", "unpacked"] = "binary"
|
||||
VLLM_USE_V2_MODEL_RUNNER: bool | None = None
|
||||
VLLM_LOG_MODEL_INSPECTION: bool = False
|
||||
@@ -323,11 +288,9 @@ if TYPE_CHECKING:
|
||||
VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: bool = True
|
||||
VLLM_XPU_FORCE_N_CONTIG_WEIGHT: bool = False
|
||||
VLLM_XPU_USE_SAMPLER_KERNEL: bool = True
|
||||
VLLM_XPU_INC_WNA16_BACKEND: Literal["auto", "ark", "w4a16", "w4a8"] = "auto"
|
||||
VLLM_LORA_ENABLE_DUAL_STREAM: bool = False
|
||||
VLLM_GPU_NIC_PCIE_MAPPING: str = ""
|
||||
VLLM_NIC_SELECTION_VARS: str = ""
|
||||
VLLM_ENABLE_HPC_OPS: bool = False
|
||||
VLLM_REQUEST_BODY_LOG_LEVEL: str = "DEBUG"
|
||||
|
||||
|
||||
@@ -595,9 +558,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# Enable batch-invariant mode: deterministic results regardless of
|
||||
# batch composition. Requires NVIDIA GPU with compute capability >= 9.0.
|
||||
"VLLM_BATCH_INVARIANT": lambda: bool(int(os.getenv("VLLM_BATCH_INVARIANT", "0"))),
|
||||
"VLLM_REPLICATE_EMBED": lambda: (
|
||||
os.getenv("VLLM_REPLICATE_EMBED", "0").strip().lower() in ("1", "true")
|
||||
),
|
||||
# Use tensor descriptors for Q/K/V loads and output stores in the
|
||||
# Triton unified-attention kernel. Enables HW 2D block reads on
|
||||
# Intel XPU; the non-TD branch is dead-code-eliminated at Triton
|
||||
@@ -640,10 +600,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
),
|
||||
# Build provenance metadata embedded in official vllm-openai images.
|
||||
# Set via Docker ENV at image build time; informational only.
|
||||
"VLLM_BUILD_COMMIT": lambda: os.environ.get("VLLM_BUILD_COMMIT", "unknown"),
|
||||
"VLLM_BUILD_PIPELINE": lambda: os.environ.get("VLLM_BUILD_PIPELINE", "local"),
|
||||
"VLLM_BUILD_URL": lambda: os.environ.get("VLLM_BUILD_URL", ""),
|
||||
"VLLM_IMAGE_TAG": lambda: os.environ.get("VLLM_IMAGE_TAG", ""),
|
||||
# CMake build type
|
||||
# If not set, defaults to "Debug" or "RelWithDebInfo"
|
||||
# Available options: "Debug", "Release", "RelWithDebInfo"
|
||||
@@ -839,13 +795,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
),
|
||||
# Pipeline stage partition strategy
|
||||
"VLLM_PP_LAYER_PARTITION": lambda: os.getenv("VLLM_PP_LAYER_PARTITION", None),
|
||||
# (CPU backend only) CPU key-value cache space.
|
||||
# default is None and will be set as 4 GB
|
||||
"VLLM_CPU_KVCACHE_SPACE": lambda: (
|
||||
int(os.getenv("VLLM_CPU_KVCACHE_SPACE", "0"))
|
||||
if "VLLM_CPU_KVCACHE_SPACE" in os.environ
|
||||
else None
|
||||
),
|
||||
# (CPU backend only) CPU core ids bound by OpenMP threads, e.g., "0-31",
|
||||
# "0,1,2", "0-31,33". CPU cores of different ranks are separated by '|'.
|
||||
"VLLM_CPU_OMP_THREADS_BIND": lambda: os.getenv("VLLM_CPU_OMP_THREADS_BIND", "auto"),
|
||||
@@ -856,12 +805,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
if "VLLM_CPU_NUM_OF_RESERVED_CPU" in os.environ
|
||||
else None
|
||||
),
|
||||
# (CPU backend only) whether vLLM is running in a CI environment.
|
||||
"VLLM_CPU_CI_ENV": lambda: bool(int(os.getenv("VLLM_CPU_CI_ENV", "0"))),
|
||||
# (CPU backend only) whether to enable attention spilt KV.
|
||||
"VLLM_CPU_ATTN_SPLIT_KV": lambda: bool(
|
||||
int(os.getenv("VLLM_CPU_ATTN_SPLIT_KV", "1"))
|
||||
),
|
||||
# (Zen CPU backend) eagerly prepack weights into ZenDNN blocked layout
|
||||
# at model load time. Eliminates per-inference layout conversion overhead.
|
||||
"VLLM_ZENTORCH_WEIGHT_PREPACK": lambda: bool(
|
||||
@@ -1025,16 +968,7 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_MEDIA_CONNECTOR": lambda: os.getenv("VLLM_MEDIA_CONNECTOR", "http"),
|
||||
# Path to the XLA persistent cache directory.
|
||||
# Only used for XLA devices such as TPUs.
|
||||
"VLLM_XLA_CACHE_PATH": lambda: os.path.expanduser(
|
||||
os.getenv(
|
||||
"VLLM_XLA_CACHE_PATH",
|
||||
os.path.join(get_default_cache_root(), "vllm", "xla_cache"),
|
||||
)
|
||||
),
|
||||
# If set, assert on XLA recompilation after each execution step.
|
||||
"VLLM_XLA_CHECK_RECOMPILATION": lambda: bool(
|
||||
int(os.getenv("VLLM_XLA_CHECK_RECOMPILATION", "0"))
|
||||
),
|
||||
# Enable SPMD mode for TPU backend.
|
||||
"VLLM_XLA_USE_SPMD": lambda: bool(int(os.getenv("VLLM_XLA_USE_SPMD", "0"))),
|
||||
# Maximum size (in MB) for logits tensor in sparse MLA indexer prefill chunks.
|
||||
@@ -1173,9 +1107,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
else os.environ["VLLM_DISABLED_KERNELS"].split(",")
|
||||
),
|
||||
# Selects hw-agnostic layers for HF transformer backend
|
||||
"VLLM_USE_HW_AGNOSTIC": lambda: (
|
||||
os.getenv("VLLM_USE_HW_AGNOSTIC", "False").lower() in ("true", "1")
|
||||
),
|
||||
"VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE": lambda: bool(
|
||||
int(os.getenv("VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE", "1"))
|
||||
),
|
||||
@@ -1277,12 +1208,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# context parallelism: "asm" uses AITER's round-robin ASM decode (gfx950),
|
||||
# "segmented" the Triton segmented MLA path. "auto" (default) takes "asm"
|
||||
# wherever it can serve the shape and "segmented" otherwise.
|
||||
"VLLM_ROCM_AITER_MLA_DCP_VERIFY": env_with_choices(
|
||||
"VLLM_ROCM_AITER_MLA_DCP_VERIFY",
|
||||
"auto",
|
||||
["auto", "asm", "segmented"],
|
||||
case_sensitive=False,
|
||||
),
|
||||
# Small-head (<16) AITER MLA decode kernel selection. Small head counts
|
||||
# (e.g. Kimi-K3: 12 heads/rank at TP8, 6 at TP16) can decode either through
|
||||
# the Gluon small-head kernel or through the padded persistent-scheduling
|
||||
@@ -1291,12 +1216,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# PS ASM decode; "gluon" forces the Gluon path wherever a build exists;
|
||||
# "asm" forces the padded PS ASM decode. On gfx942/CDNA3 there is no Gluon
|
||||
# build, so the ASM path is always used regardless of this setting.
|
||||
"VLLM_ROCM_AITER_MLA_ASM_PADDING": env_with_choices(
|
||||
"VLLM_ROCM_AITER_MLA_ASM_PADDING",
|
||||
"auto",
|
||||
["auto", "gluon", "asm"],
|
||||
case_sensitive=False,
|
||||
),
|
||||
# Whether to use aiter mha ops.
|
||||
# By default is enabled.
|
||||
"VLLM_ROCM_USE_AITER_MHA": lambda: (
|
||||
@@ -1439,9 +1358,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# Deprecated and ignored: RayExecutorV2 is the only Ray executor. Kept so
|
||||
# out-of-tree platforms that still read it keep working; will be removed
|
||||
# in v0.34.
|
||||
"VLLM_USE_RAY_V2_EXECUTOR_BACKEND": lambda: bool(
|
||||
int(os.getenv("VLLM_USE_RAY_V2_EXECUTOR_BACKEND", "1"))
|
||||
),
|
||||
# In some system, find_loaded_library() may not work. So we allow users to
|
||||
# specify the path through environment variable VLLM_CUDART_SO_PATH.
|
||||
"VLLM_CUDART_SO_PATH": lambda: os.getenv("VLLM_CUDART_SO_PATH", None),
|
||||
@@ -1546,9 +1462,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
os.environ.get("VLLM_V1_USE_OUTLINES_CACHE", "0") == "1"
|
||||
),
|
||||
# Whether using Pathways
|
||||
"VLLM_TPU_USING_PATHWAYS": lambda: bool(
|
||||
"proxy" in os.getenv("JAX_PLATFORMS", "").lower()
|
||||
),
|
||||
# Allow use of DeepGemm kernels for fused moe ops.
|
||||
"VLLM_USE_DEEP_GEMM": lambda: bool(int(os.getenv("VLLM_USE_DEEP_GEMM", "1"))),
|
||||
# Allow use of DeepGemm specifically for MoE fused ops (overrides only MoE).
|
||||
@@ -1564,7 +1477,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
int(os.getenv("VLLM_USE_DEEP_GEMM_TMA_ALIGNED_SCALES", "1"))
|
||||
),
|
||||
# Opt-in MLA DCP query replication: skip the decode query all-gather.
|
||||
"VLLM_DCP_Q_REPLICATE": lambda: bool(int(os.getenv("VLLM_DCP_Q_REPLICATE", "0"))),
|
||||
# DeepGemm JITs the kernels on-demand. The warmup attempts to make DeepGemm
|
||||
# JIT all the required kernels before model execution so there is no
|
||||
# JIT'ing in the hot-path. However, this warmup increases the engine
|
||||
@@ -1602,24 +1514,16 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# per layer, so it only wins at low token counts: intended for decode
|
||||
# instances in a P/D disaggregated deployment, not for prefill or unified
|
||||
# serving.
|
||||
"VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT": lambda: bool(
|
||||
int(os.getenv("VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT", "0"))
|
||||
),
|
||||
# Kimi K3 only, and unrelated to the MoE flags above. Tap the pre-norm
|
||||
# AttnRes mixture, rather than the post-mixture sum, as the auxiliary
|
||||
# hidden state handed to a DFlash drafter. This changes the numerics the
|
||||
# speculator sees, so it is off by default while the effect is measured.
|
||||
"VLLM_KIMI_K3_AUX_ATTN_RES_STREAM": lambda: bool(
|
||||
int(os.getenv("VLLM_KIMI_K3_AUX_ATTN_RES_STREAM", "0"))
|
||||
),
|
||||
# Use the SM100 BF16 GEMM-AR kernel for eligible Kimi-K3 row-parallel
|
||||
# attention projections. All TP ranks must belong to one NVLink domain.
|
||||
"VLLM_KIMI_K3_GEMM_AR": lambda: bool(int(os.getenv("VLLM_KIMI_K3_GEMM_AR", "1"))),
|
||||
# Fuse eligible sequence-parallel row-parallel projections with their TP
|
||||
# reduce-scatter using the SM100 BF16/MXFP8 GEMM-RS kernel (Kimi-K3
|
||||
# attention/shared-expert projections, DeepSeek-V4.1 ``wo_b``). All TP
|
||||
# ranks must belong to one NVLink domain.
|
||||
"VLLM_ENABLE_GEMM_RS": lambda: bool(int(os.getenv("VLLM_ENABLE_GEMM_RS", "0"))),
|
||||
# Allow use of FlashInfer FP8 block-scale GEMM for linear layers.
|
||||
# This uses TensorRT-LLM kernels and requires SM90+ (Hopper).
|
||||
"VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": lambda: bool(
|
||||
@@ -1666,13 +1570,7 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
int(os.getenv("VLLM_DISABLE_REQUEST_ID_RANDOMIZATION", "0"))
|
||||
),
|
||||
# IP address used for NIXL handshake between remote agents.
|
||||
"VLLM_NIXL_SIDE_CHANNEL_HOST": lambda: os.getenv(
|
||||
"VLLM_NIXL_SIDE_CHANNEL_HOST", "localhost"
|
||||
),
|
||||
# Port used for NIXL handshake between remote agents.
|
||||
"VLLM_NIXL_SIDE_CHANNEL_PORT": lambda: int(
|
||||
os.getenv("VLLM_NIXL_SIDE_CHANNEL_PORT", "5600")
|
||||
),
|
||||
# Address the P2P KV-offload control socket binds to. Defaults to
|
||||
# ``localhost`` (loopback only); must be set to the node IP for
|
||||
# cross-host P2P so remote peers can reach the socket.
|
||||
@@ -1700,39 +1598,18 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_P2P_IDLE_TIMEOUT_S": lambda: int(os.getenv("VLLM_P2P_IDLE_TIMEOUT_S", "300")),
|
||||
# IP address used for the EC connector's ZMQ side channel
|
||||
# (producer ROUTER bind, consumer DEALER dial).
|
||||
"VLLM_EC_SIDE_CHANNEL_HOST": lambda: os.getenv(
|
||||
"VLLM_EC_SIDE_CHANNEL_HOST", "localhost"
|
||||
),
|
||||
# Port for the EC connector's ZMQ side channel; advertised to peers
|
||||
# via `ec_transfer_params.peer_port` on the producer's response.
|
||||
"VLLM_EC_SIDE_CHANNEL_PORT": lambda: int(
|
||||
os.getenv("VLLM_EC_SIDE_CHANNEL_PORT", "5601")
|
||||
),
|
||||
# Per-request timeout (seconds) when a prefiller worker registers with
|
||||
# the Mooncake bootstrap server.
|
||||
"VLLM_MOONCAKE_CONNECTOR_TIMEOUT": lambda: float(
|
||||
os.getenv("VLLM_MOONCAKE_CONNECTOR_TIMEOUT", "30.0")
|
||||
),
|
||||
# Port used for Mooncake handshake between remote agents.
|
||||
"VLLM_MOONCAKE_BOOTSTRAP_PORT": lambda: int(
|
||||
os.getenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", "8998")
|
||||
),
|
||||
# Log per-batch memory/disk tier breakdown on external GETs.
|
||||
"VLLM_MOONCAKE_STORE_TIER_LOG": lambda: (
|
||||
os.getenv("VLLM_MOONCAKE_STORE_TIER_LOG", "False").lower() in ("true", "1")
|
||||
),
|
||||
# Number of parallel KV-load receive threads per worker rank. Lets the
|
||||
# per-request control overhead (Python prep + master key lookup) of one
|
||||
# request overlap with the RDMA transfer of another, keeping the transfer
|
||||
# engine's queue pairs busy. Helps when that overhead is significant or
|
||||
# per-request batches are too small to saturate the link on their own.
|
||||
"VLLM_MOONCAKE_LOAD_RECV_THREADS": lambda: int(
|
||||
os.getenv("VLLM_MOONCAKE_LOAD_RECV_THREADS", "1")
|
||||
),
|
||||
# Fraction of the owner's DirectIO staging buffer to fill per GET batch.
|
||||
"VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO": lambda: float(
|
||||
os.getenv("VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO", "0.9")
|
||||
),
|
||||
# Pin this rank to a specific owner segment ("host:port").
|
||||
"MOONCAKE_PREFERRED_SEGMENT": lambda: os.getenv("MOONCAKE_PREFERRED_SEGMENT"),
|
||||
# Override the hostname the rank registers as a Mooncake requester.
|
||||
@@ -1788,18 +1665,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_MOE_ROUTING_SIMULATION_STRATEGY", ""
|
||||
).lower(),
|
||||
# Regex timeout for use by the vLLM tool parsing plugins.
|
||||
"VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS": lambda: int(
|
||||
os.getenv("VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS", "1")
|
||||
),
|
||||
# Enforce function parameter schemas in structural-tag based tool calling.
|
||||
"VLLM_ENFORCE_STRICT_TOOL_CALLING": lambda: (
|
||||
os.getenv("VLLM_ENFORCE_STRICT_TOOL_CALLING", "True").lower() in ("true", "1")
|
||||
),
|
||||
# Constrain every `beam` output to its response template's format, also
|
||||
# without tool or structured output constraints.
|
||||
"VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT": lambda: (
|
||||
os.getenv("VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT", "True").lower() in ("true", "1")
|
||||
),
|
||||
# Control the max chunk bytes (in MB) for the rpc message queue.
|
||||
# Object larger than this threshold will be broadcast to worker
|
||||
# processes via zmq.
|
||||
@@ -1849,9 +1720,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
int(os.getenv("VLLM_RAISE_ON_LOGIT_NANS", "0"))
|
||||
),
|
||||
# Timeout (in seconds) for MooncakeConnector in PD disaggregated setup.
|
||||
"VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT": lambda: int(
|
||||
os.getenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "480")
|
||||
),
|
||||
# If set, it means we pre-downloaded cubin files and flashinfer will
|
||||
# read the cubin files directly.
|
||||
"VLLM_HAS_FLASHINFER_CUBIN": lambda: bool(
|
||||
@@ -1891,12 +1759,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_ENABLE_RESPONSES_API_STORE": lambda: bool(
|
||||
int(os.getenv("VLLM_ENABLE_RESPONSES_API_STORE", "0"))
|
||||
),
|
||||
# If set to 1, expose the Cohere Chat v2 API at ``POST /cohere/v2/chat``
|
||||
# and its render endpoint at ``POST /cohere/v2/chat/render``.
|
||||
# Default off
|
||||
"VLLM_ENABLE_COHERE_API": lambda: bool(
|
||||
int(os.getenv("VLLM_ENABLE_COHERE_API", "0"))
|
||||
),
|
||||
# If set, use the fp8 mfma in rocm paged attention.
|
||||
"VLLM_ROCM_FP8_MFMA_PAGE_ATTN": lambda: bool(
|
||||
int(os.getenv("VLLM_ROCM_FP8_MFMA_PAGE_ATTN", "0"))
|
||||
@@ -2056,9 +1918,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# Emergency rollback for the DeepSeek-V4 NVIDIA MegaMoE path. By default,
|
||||
# DeepGEMM computes replicated FP8 shared experts in the same persistent
|
||||
# SM100 kernel as the routed FP4 experts.
|
||||
"VLLM_DISABLE_DSV4_MEGAMOE_SHARED_EXPERT_FUSION": lambda: bool(
|
||||
int(os.getenv("VLLM_DISABLE_DSV4_MEGAMOE_SHARED_EXPERT_FUSION", "0"))
|
||||
),
|
||||
# Limits when we run shared_experts in a separate stream.
|
||||
# We found out that for large batch sizes, the separate stream
|
||||
# execution is not beneficial (most likely because of the input clone)
|
||||
@@ -2074,9 +1933,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# and the cross-stream sync becomes pure overhead. Set to 0 to disable
|
||||
# the multi-stream path entirely. See #PR 41526 for the empirical result
|
||||
# for the default value of 1024 tokens.
|
||||
"VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD": lambda: int(
|
||||
os.getenv("VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD", "1024")
|
||||
),
|
||||
# Format for saving torch.compile cache artifacts
|
||||
# - "binary": saves as binary file
|
||||
# Safe for multiple vllm serve processes accessing the same torch compile cache.
|
||||
@@ -2175,9 +2031,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# activations to per-token int8 (int4_gemm_w4a8) for large token counts.
|
||||
# The two oneDNN backends are int4-only; ARK also serves int2.
|
||||
# Which one is fastest is device-dependent, so this is left as an opt-in.
|
||||
"VLLM_XPU_INC_WNA16_BACKEND": env_with_choices(
|
||||
"VLLM_XPU_INC_WNA16_BACKEND", "auto", ["auto", "ark", "w4a16", "w4a8"]
|
||||
),
|
||||
# Direct DCP ops default on when applicable; set to 1 to enforce or 0 to disable.
|
||||
"VLLM_USE_DIRECT_DCP_A2A": lambda: maybe_convert_bool(
|
||||
os.getenv("VLLM_USE_DIRECT_DCP_A2A")
|
||||
@@ -2212,7 +2065,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# fused into gated_mla_gemm; elementwise gating only.
|
||||
# Each op additionally checks its own shape / dtype constraints and falls
|
||||
# back to the eager path when they do not hold.
|
||||
"VLLM_ENABLE_HPC_OPS": lambda: bool(int(os.getenv("VLLM_ENABLE_HPC_OPS", "0"))),
|
||||
# Whether to skip version suffix when building package
|
||||
"VLLM_SKIP_VERSION_SUFFIX": lambda: bool(
|
||||
int(os.getenv("VLLM_SKIP_VERSION_SUFFIX", "0"))
|
||||
@@ -2308,13 +2160,11 @@ def compile_factors() -> dict[str, object]:
|
||||
# HOME or the XDG roots silently invalidates every compile cache
|
||||
# (VLLM_CACHE_ROOT above and VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR below
|
||||
# are already ignored for the same reason).
|
||||
"VLLM_XLA_CACHE_PATH",
|
||||
"VLLM_CONFIG_ROOT",
|
||||
"LD_LIBRARY_PATH",
|
||||
"VLLM_SERVER_DEV_MODE",
|
||||
"VLLM_DP_MASTER_IP",
|
||||
"VLLM_DP_MASTER_PORT",
|
||||
"VLLM_NIXL_SIDE_CHANNEL_HOST",
|
||||
"VLLM_RANDOMIZE_DP_DUMMY_INPUTS",
|
||||
"VLLM_MODEL_REDIRECT_PATH",
|
||||
"VLLM_HOST_IP",
|
||||
@@ -2369,7 +2219,6 @@ def compile_factors() -> dict[str, object]:
|
||||
"VLLM_WORKER_MULTIPROC_METHOD",
|
||||
"VLLM_ENABLE_V1_MULTIPROCESSING",
|
||||
"VLLM_V1_OUTPUT_PROC_CHUNK_SIZE",
|
||||
"VLLM_CPU_KVCACHE_SPACE",
|
||||
"VLLM_CPU_MOE_PREPACK",
|
||||
"VLLM_ZENTORCH_WEIGHT_PREPACK",
|
||||
"VLLM_TEST_FORCE_LOAD_FORMAT",
|
||||
|
||||
Reference in New Issue
Block a user