[Mini-vLLM] SM86-only cut: models, backends, Rust, tests
- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries - Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels - Remove inc/quark quant loaders; keep compressed-tensors and Marlin - Drop the Rust frontend, pooling entrypoints, and idle test suites - Rebuild kernels for sm_80;sm_86 and strip the extensions - Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
+1
-177
@@ -2,7 +2,6 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
import contextlib
|
||||
import json
|
||||
import multiprocessing
|
||||
import threading
|
||||
import time
|
||||
@@ -277,129 +276,6 @@ class APIServerProcessManager:
|
||||
)
|
||||
|
||||
|
||||
class RustFrontendProcessManager:
|
||||
"""Manages a single Rust frontend subprocess.
|
||||
|
||||
Launches the Rust vllm-rs binary in 'frontend' mode, passing the
|
||||
HTTP and ZMQ listener file descriptors. Provides the same
|
||||
interface as APIServerProcessManager for process monitoring.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
binary_path: str,
|
||||
sock: Any,
|
||||
args: argparse.Namespace,
|
||||
input_listener: ZmqListener,
|
||||
output_listener: ZmqListener,
|
||||
engine_start_index: int,
|
||||
engine_count: int,
|
||||
data_parallel_size: int,
|
||||
stats_update_address: str | None = None,
|
||||
grpc_sock: Any | None = None,
|
||||
):
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
listen_fd = sock.fileno()
|
||||
input_fd = input_listener.socket.fileno()
|
||||
output_fd = output_listener.socket.fileno()
|
||||
inherited_fds = [listen_fd, input_fd, output_fd]
|
||||
for fd in inherited_fds:
|
||||
os.set_inheritable(fd, True)
|
||||
|
||||
cmd = [
|
||||
binary_path,
|
||||
"frontend",
|
||||
"--listen-fd",
|
||||
str(listen_fd),
|
||||
"--input-listener-fd",
|
||||
str(input_fd),
|
||||
"--output-listener-fd",
|
||||
str(output_fd),
|
||||
"--engine-start-index",
|
||||
str(engine_start_index),
|
||||
"--engine-count",
|
||||
str(engine_count),
|
||||
"--data-parallel-size",
|
||||
str(data_parallel_size),
|
||||
]
|
||||
if grpc_sock is not None:
|
||||
grpc_fd = grpc_sock.fileno()
|
||||
os.set_inheritable(grpc_fd, True)
|
||||
cmd.extend(["--grpc-listen-fd", str(grpc_fd)])
|
||||
inherited_fds.append(grpc_fd)
|
||||
if stats_update_address is not None:
|
||||
cmd.extend(["--coordinator-address", stats_update_address])
|
||||
from vllm.entrypoints.serve.utils.api_utils import jsonify_non_default_args
|
||||
|
||||
args_dict = jsonify_non_default_args(
|
||||
args,
|
||||
exclude={
|
||||
"api_server_count",
|
||||
# Python passes the bootstrapped engine range explicitly.
|
||||
"data_parallel_rank",
|
||||
"data_parallel_external_lb",
|
||||
"data_parallel_hybrid_lb",
|
||||
},
|
||||
)
|
||||
|
||||
# `model_tag` is the positional `vllm serve` model argument. When the
|
||||
# model is supplied only through `--config`, argparse populates `model`
|
||||
# while leaving `model_tag` unset. The Rust frontend requires
|
||||
# `model_tag` in its JSON bootstrap payload, so use the resolved model
|
||||
# as a fallback.
|
||||
model_tag = getattr(args, "model_tag", None) or getattr(args, "model", None)
|
||||
if model_tag is not None:
|
||||
args_dict["model_tag"] = model_tag
|
||||
|
||||
# The Rust `frontend` subcommand parses --args-json via serde_json,
|
||||
# which bypasses clap and therefore ignores any `#[arg(env = ...)]`
|
||||
# declarations on SharedRuntimeArgs fields. Forward the env-driven
|
||||
# values explicitly so VLLM_ENGINE_READY_TIMEOUT_S and
|
||||
# VLLM_HTTP_TIMEOUT_KEEP_ALIVE behave the same on both Python and Rust
|
||||
# frontends.
|
||||
args_dict["engine_ready_timeout_secs"] = envs.VLLM_ENGINE_READY_TIMEOUT_S
|
||||
args_dict["http_timeout_keep_alive"] = envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE
|
||||
args_json = json.dumps(args_dict, sort_keys=True)
|
||||
cmd.extend(["--args-json", args_json])
|
||||
|
||||
# The subprocess needs the real values, but the log must not carry
|
||||
# credentials such as api_key or hf_token.
|
||||
from vllm.entrypoints.serve.utils.api_utils import redact_sensitive_args
|
||||
|
||||
redacted_json = json.dumps(redact_sensitive_args(args_dict), sort_keys=True)
|
||||
logger.info("Launching Rust frontend: %s", " ".join(cmd[:-1] + [redacted_json]))
|
||||
self._listeners = [input_listener, output_listener]
|
||||
try:
|
||||
self._proc = subprocess.Popen(cmd, pass_fds=inherited_fds)
|
||||
except BaseException:
|
||||
for listener in self._listeners:
|
||||
listener.cleanup()
|
||||
raise
|
||||
else:
|
||||
input_listener.close()
|
||||
output_listener.close()
|
||||
|
||||
# Create a process wrapper with a sentinel fd for monitoring
|
||||
self.processes: list[_SubprocessWrapper] = [
|
||||
_SubprocessWrapper(self._proc, "RustFrontend")
|
||||
]
|
||||
|
||||
self._finalizer = weakref.finalize(
|
||||
self,
|
||||
_shutdown_subprocesses_and_cleanup_listeners,
|
||||
self.processes,
|
||||
self._listeners,
|
||||
)
|
||||
|
||||
def shutdown(self, timeout: float | None = None) -> None:
|
||||
if self._finalizer.detach() is not None:
|
||||
_shutdown_subprocesses_and_cleanup_listeners(
|
||||
self.processes, self._listeners, timeout=timeout
|
||||
)
|
||||
|
||||
|
||||
class _SubprocessWrapper:
|
||||
"""Wraps subprocess.Popen to provide the BaseProcess-like interface
|
||||
needed by wait_for_completion_or_failure."""
|
||||
@@ -454,58 +330,6 @@ class _SubprocessWrapper:
|
||||
self._sentinel_send.close()
|
||||
|
||||
|
||||
def _shutdown_subprocesses(
|
||||
procs: list[_SubprocessWrapper], timeout: float | None = None
|
||||
) -> None:
|
||||
"""Shutdown subprocess wrappers (mirrors the shutdown() function)."""
|
||||
if timeout is None:
|
||||
timeout = 0.0
|
||||
timeout = max(timeout, 5.0)
|
||||
|
||||
logger.debug(
|
||||
"[shutdown] Subprocess manager: start process_count=%d timeout=%ss",
|
||||
len(procs),
|
||||
timeout,
|
||||
)
|
||||
|
||||
for proc in procs:
|
||||
if proc.is_alive():
|
||||
proc.terminate()
|
||||
|
||||
deadline = time.monotonic() + timeout
|
||||
for proc in procs:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
break
|
||||
if proc.is_alive():
|
||||
proc.join(remaining)
|
||||
|
||||
remaining_pids = [
|
||||
proc.pid for proc in procs if proc.is_alive() and proc.pid is not None
|
||||
]
|
||||
if remaining_pids:
|
||||
logger.warning(
|
||||
"[shutdown] Subprocess manager: force killing remaining processes count=%d",
|
||||
len(remaining_pids),
|
||||
)
|
||||
for pid in remaining_pids:
|
||||
kill_process_tree(pid)
|
||||
|
||||
logger.debug_once("[shutdown] Subprocess manager: complete")
|
||||
|
||||
|
||||
def _shutdown_subprocesses_and_cleanup_listeners(
|
||||
procs: list[_SubprocessWrapper],
|
||||
listeners: list[ZmqListener],
|
||||
timeout: float | None = None,
|
||||
) -> None:
|
||||
try:
|
||||
_shutdown_subprocesses(procs, timeout=timeout)
|
||||
finally:
|
||||
for listener in listeners:
|
||||
listener.cleanup()
|
||||
|
||||
|
||||
def run_api_server_worker_proc(
|
||||
listen_address, sock, args, client_config=None, **uvicorn_kwargs
|
||||
) -> None:
|
||||
@@ -530,7 +354,7 @@ def run_api_server_worker_proc(
|
||||
|
||||
|
||||
def wait_for_completion_or_failure(
|
||||
api_server_manager: "APIServerProcessManager | RustFrontendProcessManager",
|
||||
api_server_manager: "APIServerProcessManager",
|
||||
engine_manager: Union["CoreEngineProcManager", "CoreEngineActorManager"]
|
||||
| None = None,
|
||||
coordinator: "DPCoordinator | None" = None,
|
||||
|
||||
Reference in New Issue
Block a user