[Frontend][RFC] Rust front-end integration (#40848)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
+51
-1
@@ -86,6 +86,7 @@ if TYPE_CHECKING:
|
||||
MAX_JOBS: str | None = None
|
||||
NVCC_THREADS: str | None = None
|
||||
VLLM_USE_PRECOMPILED: bool = False
|
||||
VLLM_USE_PRECOMPILED_RUST: bool = False
|
||||
VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX: bool = False
|
||||
VLLM_DOCKER_BUILD_CONTEXT: bool = False
|
||||
VLLM_KEEP_ALIVE_ON_ENGINE_DEATH: bool = False
|
||||
@@ -134,6 +135,8 @@ if TYPE_CHECKING:
|
||||
Q_SCALE_CONSTANT: int = 200
|
||||
K_SCALE_CONSTANT: int = 200
|
||||
V_SCALE_CONSTANT: int = 100
|
||||
VLLM_USE_RUST_FRONTEND: bool = False
|
||||
VLLM_RUST_FRONTEND_PATH: str | None = "auto"
|
||||
VLLM_SERVER_DEV_MODE: bool = False
|
||||
VLLM_V1_OUTPUT_PROC_CHUNK_SIZE: int = 128
|
||||
VLLM_MLA_DISABLE: bool = False
|
||||
@@ -514,6 +517,40 @@ def get_env_or_set_default(
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _resolve_rust_frontend_path() -> str | None:
|
||||
"""Resolve the Rust frontend binary path.
|
||||
|
||||
Returns None if VLLM_USE_RUST_FRONTEND is not enabled.
|
||||
When enabled, resolves VLLM_RUST_FRONTEND_PATH ("auto" by default)
|
||||
to the actual binary path.
|
||||
"""
|
||||
use_rust = bool(int(os.environ.get("VLLM_USE_RUST_FRONTEND", "0")))
|
||||
raw = os.environ.get("VLLM_RUST_FRONTEND_PATH", "auto")
|
||||
|
||||
if not use_rust:
|
||||
if os.environ.get("VLLM_RUST_FRONTEND_PATH") is not None:
|
||||
logger.warning(
|
||||
"VLLM_RUST_FRONTEND_PATH is set but VLLM_USE_RUST_FRONTEND "
|
||||
"is not enabled. The Rust frontend will not be used. "
|
||||
"Set VLLM_USE_RUST_FRONTEND=1 to enable it."
|
||||
)
|
||||
return None
|
||||
|
||||
if raw.lower() in ("auto", "1", "true"):
|
||||
pkg_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
candidate = os.path.join(pkg_dir, "vllm-rs")
|
||||
if os.path.isfile(candidate) and os.access(candidate, os.X_OK):
|
||||
return candidate
|
||||
|
||||
raise FileNotFoundError(
|
||||
"VLLM_RUST_FRONTEND_PATH=auto but the vllm-rs binary was "
|
||||
f"not found at {candidate}. "
|
||||
"Build with setuptools-rust or set the path explicitly."
|
||||
)
|
||||
return raw
|
||||
|
||||
|
||||
environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# ================== Installation Time Env Vars ==================
|
||||
# Target device of vLLM, supporting [cuda (by default),
|
||||
@@ -551,11 +588,15 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# By default this is 1.
|
||||
# If set, `MAX_JOBS` will be reduced to avoid oversubscribing the CPU.
|
||||
"NVCC_THREADS": lambda: os.getenv("NVCC_THREADS", None),
|
||||
# If set, vllm will use precompiled binaries (*.so)
|
||||
# If set, vllm will use precompiled native binaries (*.so and vllm-rs).
|
||||
"VLLM_USE_PRECOMPILED": lambda: (
|
||||
os.environ.get("VLLM_USE_PRECOMPILED", "").strip().lower() in ("1", "true")
|
||||
or bool(os.environ.get("VLLM_PRECOMPILED_WHEEL_LOCATION"))
|
||||
),
|
||||
# If set, vllm will use the precompiled Rust frontend binary (vllm-rs).
|
||||
"VLLM_USE_PRECOMPILED_RUST": lambda: (
|
||||
os.environ.get("VLLM_USE_PRECOMPILED_RUST", "").strip().lower() in ("1", "true")
|
||||
),
|
||||
# If set, skip adding +precompiled suffix to version string
|
||||
"VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX": lambda: bool(
|
||||
int(os.environ.get("VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX", "0"))
|
||||
@@ -1154,6 +1195,15 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# If set to "0", disable LayerName opaque type for layer_name
|
||||
# parameters in custom ops. Defaults to enabled on torch >= 2.11.
|
||||
"VLLM_USE_LAYERNAME": lambda: bool(int(os.getenv("VLLM_USE_LAYERNAME", "1"))),
|
||||
# If set, use the Rust frontend binary instead of the Python API server
|
||||
# process(es).
|
||||
"VLLM_USE_RUST_FRONTEND": lambda: bool(
|
||||
int(os.getenv("VLLM_USE_RUST_FRONTEND", "0"))
|
||||
),
|
||||
# Path to the Rust frontend binary. Defaults to "auto" which discovers
|
||||
# the binary installed with the vllm package. Only used when
|
||||
# VLLM_USE_RUST_FRONTEND=1.
|
||||
"VLLM_RUST_FRONTEND_PATH": lambda: _resolve_rust_frontend_path(),
|
||||
# If set, vllm will run in development mode, which will enable
|
||||
# some additional endpoints for developing and debugging,
|
||||
# e.g. `/reset_prefix_cache`
|
||||
|
||||
Reference in New Issue
Block a user