[Frontend][RFC] Rust front-end integration (#40848)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Nick Hill
2026-05-21 12:24:48 +08:00
committed by GitHub
co-authored by Bugen Zhao
parent d97ba29fdc
commit f2ace1d57d
35 changed files with 893 additions and 91 deletions
+51 -1
View File
@@ -86,6 +86,7 @@ if TYPE_CHECKING:
MAX_JOBS: str | None = None
NVCC_THREADS: str | None = None
VLLM_USE_PRECOMPILED: bool = False
VLLM_USE_PRECOMPILED_RUST: bool = False
VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX: bool = False
VLLM_DOCKER_BUILD_CONTEXT: bool = False
VLLM_KEEP_ALIVE_ON_ENGINE_DEATH: bool = False
@@ -134,6 +135,8 @@ if TYPE_CHECKING:
Q_SCALE_CONSTANT: int = 200
K_SCALE_CONSTANT: int = 200
V_SCALE_CONSTANT: int = 100
VLLM_USE_RUST_FRONTEND: bool = False
VLLM_RUST_FRONTEND_PATH: str | None = "auto"
VLLM_SERVER_DEV_MODE: bool = False
VLLM_V1_OUTPUT_PROC_CHUNK_SIZE: int = 128
VLLM_MLA_DISABLE: bool = False
@@ -514,6 +517,40 @@ def get_env_or_set_default(
logger = logging.getLogger(__name__)
def _resolve_rust_frontend_path() -> str | None:
"""Resolve the Rust frontend binary path.
Returns None if VLLM_USE_RUST_FRONTEND is not enabled.
When enabled, resolves VLLM_RUST_FRONTEND_PATH ("auto" by default)
to the actual binary path.
"""
use_rust = bool(int(os.environ.get("VLLM_USE_RUST_FRONTEND", "0")))
raw = os.environ.get("VLLM_RUST_FRONTEND_PATH", "auto")
if not use_rust:
if os.environ.get("VLLM_RUST_FRONTEND_PATH") is not None:
logger.warning(
"VLLM_RUST_FRONTEND_PATH is set but VLLM_USE_RUST_FRONTEND "
"is not enabled. The Rust frontend will not be used. "
"Set VLLM_USE_RUST_FRONTEND=1 to enable it."
)
return None
if raw.lower() in ("auto", "1", "true"):
pkg_dir = os.path.dirname(os.path.abspath(__file__))
candidate = os.path.join(pkg_dir, "vllm-rs")
if os.path.isfile(candidate) and os.access(candidate, os.X_OK):
return candidate
raise FileNotFoundError(
"VLLM_RUST_FRONTEND_PATH=auto but the vllm-rs binary was "
f"not found at {candidate}. "
"Build with setuptools-rust or set the path explicitly."
)
return raw
environment_variables: dict[str, Callable[[], Any]] = {
# ================== Installation Time Env Vars ==================
# Target device of vLLM, supporting [cuda (by default),
@@ -551,11 +588,15 @@ environment_variables: dict[str, Callable[[], Any]] = {
# By default this is 1.
# If set, `MAX_JOBS` will be reduced to avoid oversubscribing the CPU.
"NVCC_THREADS": lambda: os.getenv("NVCC_THREADS", None),
# If set, vllm will use precompiled binaries (*.so)
# If set, vllm will use precompiled native binaries (*.so and vllm-rs).
"VLLM_USE_PRECOMPILED": lambda: (
os.environ.get("VLLM_USE_PRECOMPILED", "").strip().lower() in ("1", "true")
or bool(os.environ.get("VLLM_PRECOMPILED_WHEEL_LOCATION"))
),
# If set, vllm will use the precompiled Rust frontend binary (vllm-rs).
"VLLM_USE_PRECOMPILED_RUST": lambda: (
os.environ.get("VLLM_USE_PRECOMPILED_RUST", "").strip().lower() in ("1", "true")
),
# If set, skip adding +precompiled suffix to version string
"VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX": lambda: bool(
int(os.environ.get("VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX", "0"))
@@ -1154,6 +1195,15 @@ environment_variables: dict[str, Callable[[], Any]] = {
# If set to "0", disable LayerName opaque type for layer_name
# parameters in custom ops. Defaults to enabled on torch >= 2.11.
"VLLM_USE_LAYERNAME": lambda: bool(int(os.getenv("VLLM_USE_LAYERNAME", "1"))),
# If set, use the Rust frontend binary instead of the Python API server
# process(es).
"VLLM_USE_RUST_FRONTEND": lambda: bool(
int(os.getenv("VLLM_USE_RUST_FRONTEND", "0"))
),
# Path to the Rust frontend binary. Defaults to "auto" which discovers
# the binary installed with the vllm package. Only used when
# VLLM_USE_RUST_FRONTEND=1.
"VLLM_RUST_FRONTEND_PATH": lambda: _resolve_rust_frontend_path(),
# If set, vllm will run in development mode, which will enable
# some additional endpoints for developing and debugging,
# e.g. `/reset_prefix_cache`