MOVZXandHinata
fe3cd9247b
git am refreshes every file timestamp, so a Python-only patch series forced a full extension rebuild and ninja saw everything as changed. Diff csrc/ and the build files between the recorded tree and HEAD, and rebuild only when a C++/CUDA source moved; otherwise the shipped .so stays. Update the expected tree to the 106-patch series (c9c526755364d44d8d9923d9acc9d6d1a3d58718). Add ipc: host, SYS_PTRACE, and seccomp=unconfined to the container so a profiler or debugger can attach. Raise --block-size from 2048 to 4096 and drop --safetensors-load-strategy prefetch. Co-Authored-By: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
319 lines
19 KiB
Bash
Executable File
319 lines
19 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
# Serve Qwen3.8-Flash-Next-W4A16 on 4x RTX 3090. Builds first when the tree or the venv is
|
|
# missing, checks the box, then runs the engine. One script for bare metal and for the image.
|
|
#
|
|
# sudo ./start.sh [checkpoint-dir] [extra vllm args] # build on first run, then serve
|
|
# ./build.sh --force # compile again (see ./build.sh)
|
|
#
|
|
# Shape: TP2 x PP2 + EP, MTP K=3, 262K per request, ~925K-token FP8 KV pool, full 262K context,
|
|
# host-resident PLE/embeddings, prefix caching with a 64-token match unit, RecoverSSM, side-cache
|
|
# layout. This tree's profile guard boots ONLY the command at the end of this script;
|
|
# --enforce-eager, other cudagraph modes and breakable graphs are refused.
|
|
#
|
|
# Env:
|
|
# CKPT / MODEL_MOUNT_PATH checkpoint dir (bare metal: the Swift1.5 Qwen3.8-Flash-Next W4A16
|
|
# build; container: /models/Qwen3.8-Flash-Next). A first argument that starts with
|
|
# `-` is a vllm argument, so the default checkpoint is kept.
|
|
# TREE VENV serving tree and venv (default: ./vllm and ./.venv, built by build.sh)
|
|
# PORT (8000) HOST (127.0.0.1) MODEL_NAME ("name1 name2 ...", first is the health probe's)
|
|
# SCALES KV scale sidecar, host path (default: $CKPT/qsa_kv_scales.json)
|
|
# CACHE_ROOT compile/cudagraph cache (default: .vllm-cache next to this script, on disk)
|
|
# PLE_HOME GPU that hosts the PLE offload home (0)
|
|
# GUARD bounds-guard telemetry off|warn|raise (warn)
|
|
# COUNTERS=1 logging-only counters (top-k exact restarts /30s, FP8 KV clips /300s)
|
|
# VLLM_LOG_STATS_INTERVAL seconds between engine stats lines (1; the vLLM default is 10 s)
|
|
# MAX_SEQS (4) concurrent requests. The CUDA graph list follows it, to MAX_SEQS x 4
|
|
# tokens (MTP K=3). N x 262144 tokens above the 924,993-token pool preempts.
|
|
# MAX_BATCHED_TOKENS (2048) prefill chunk in tokens. Must be >= MAX_SEQS. 2048 matches
|
|
# --block-size; 4096 prefills one max-size image in one chunk.
|
|
# CUDA_HOME toolkit for first-serve JIT; default: the venv's CUDA 13.0 wheels
|
|
# AUTO_TOPO 0 (default, fixed shipped settings) | 1 | strict -> scripts/topo_select.py
|
|
# AUTO_TOPO_NCCL=default AUTO_TOPO_BIND=1 LANE_GPUS=auto LANE_CPUS=... TOPO_TIMEOUT=20
|
|
# LOG log file (/var/log/vllm.log). LOG= (empty) keeps the output on the terminal,
|
|
# which is what the image does so `docker compose logs` works.
|
|
#
|
|
# First serve compiles kernels (~6 min, writes $CACHE_ROOT). RAM: the PLE n-gram table is host
|
|
# memory, resident floor ~69 GiB. /dev/shm >= 1 GB. One CUDA process per GPU.
|
|
|
|
set -euo pipefail
|
|
|
|
HERE="$(cd "$(dirname "$0")" && pwd)"
|
|
TREE="${TREE:-$HERE/vllm}"; VENV="${VENV:-$HERE/.venv}"
|
|
CKPT="${CKPT:-${MODEL_MOUNT_PATH:-/data/models/halt95/Swift1.5-Qwen3.8-Flash-Next-W4A16-Merlin}}"
|
|
CACHE_ROOT="${CACHE_ROOT:-$HERE/.vllm-cache}"
|
|
# Colon-free default: LOG= (set, empty) is the container's "do not redirect".
|
|
LOG="${LOG-/var/log/vllm.log}"
|
|
|
|
if [ $# -gt 0 ]; then
|
|
case "$1" in -*) ;; "") shift ;; *) CKPT="$1"; shift ;; esac
|
|
fi
|
|
case "$TREE" in /*) ;; *) TREE="$PWD/$TREE" ;; esac
|
|
case "$VENV" in /*) ;; *) VENV="$PWD/$VENV" ;; esac
|
|
|
|
# --- 1. build when this box has no serving tree or no venv ---------------------------------------
|
|
mkdir -p "$CACHE_ROOT" 2>/dev/null || { echo "cannot create $CACHE_ROOT; set CACHE_ROOT=<path>"; exit 1; }
|
|
if mountpoint -q "$CACHE_ROOT" 2>/dev/null; then cache_kind="RAM"; else cache_kind="disk"; fi
|
|
echo "cache: $CACHE_ROOT ($cache_kind)"
|
|
if [ ! -f "$TREE/vllm/__init__.py" ] || [ ! -x "$VENV/bin/vllm" ]; then
|
|
echo "first run: building the tree and the venv (./build.sh, ~20 min on 64 cores)"
|
|
"$HERE/build.sh"
|
|
fi
|
|
|
|
# --- 2. the checkpoint ---------------------------------------------------------------------------
|
|
if [ ! -f "$CKPT/config.json" ]; then
|
|
echo "checkpoint not found at $CKPT"
|
|
echo "bare metal: sudo ./start.sh /path/to/Qwen3.8-Flash-Next-W4A16-Merlin"
|
|
echo "container: -v /path/to/Qwen3.8-Flash-Next-W4A16-Merlin:/models/Qwen3.8-Flash-Next:ro"
|
|
exit 1
|
|
fi
|
|
if ! "$VENV/bin/python" -c 'import json,sys; c=json.load(open(sys.argv[1])); sys.exit(0 if c.get("text_config",{}).get("ple_embedding_dtype")=="float8_e4m3fn" else 1)' "$CKPT/config.json"; then
|
|
if [ -w "$CKPT" ] && [ -w "$CKPT/config.json" ]; then
|
|
echo "config.json lacks text_config.ple_embedding_dtype; adding it (weights untouched, backup kept)"
|
|
python3 "$HERE/scripts/make-e1-config.py" "$CKPT"
|
|
else
|
|
echo "config.json lacks text_config.ple_embedding_dtype and the directory is read-only."
|
|
echo "Run once on the writable copy: python3 scripts/make-e1-config.py $CKPT"
|
|
exit 1
|
|
fi
|
|
fi
|
|
if ! command -v nvidia-smi >/dev/null 2>&1 || ! nvidia-smi -L >/dev/null 2>&1; then
|
|
echo "WARNING: no NVIDIA GPU visible (in Docker: --gpus all, and nvidia-ctk runtime configure --runtime=docker)"
|
|
fi
|
|
|
|
# --- 3. the build products -----------------------------------------------------------------------
|
|
[ -f "$TREE/vllm/__init__.py" ] || { echo "TREE=$TREE is not the serving tree"; exit 1; }
|
|
[ -f "$TREE/.build-products" ] && [ -s "$TREE/.build-products.sha256" ] \
|
|
|| { echo "TREE=$TREE has no verified build products (./build.sh)"; exit 1; }
|
|
(cd "$TREE" && sha256sum -c --quiet --strict .build-products.sha256) \
|
|
|| { echo "TREE=$TREE: build products changed since ./build.sh verified them"; exit 1; }
|
|
[ -x "$VENV/bin/python" ] || { echo "VENV=$VENV has no bin/python"; exit 1; }
|
|
|
|
# A vllm/ in the launch directory shadows $TREE: `python -m` puts CWD before PYTHONPATH.
|
|
if [ -f "$PWD/vllm/__init__.py" ] && [ "$(cd "$PWD/vllm" 2>/dev/null && pwd -P)" != "$(cd "$TREE/vllm" && pwd -P)" ]; then
|
|
echo "refusing to start: $PWD/vllm would shadow $TREE/vllm; start from another directory"; exit 1
|
|
fi
|
|
|
|
# --- 4. first-serve toolchain (FlashInfer JITs against CUDA_HOME, Triton against Python.h) -------
|
|
if [ -z "${CUDA_HOME:-}" ]; then
|
|
CUDA_HOME="$("$VENV/bin/python" -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')/nvidia/cu13"
|
|
fi
|
|
export CUDA_HOME PATH="$CUDA_HOME/bin:$PATH"
|
|
for t in cc c++ ninja; do command -v "$t" >/dev/null || { echo "first serve needs $t (apt install gcc g++ ninja-build)"; exit 1; }; done
|
|
"$VENV/bin/python" -c 'import os,sys,sysconfig; sys.exit(0 if os.path.exists(os.path.join(sysconfig.get_paths()["include"],"Python.h")) else 1)' \
|
|
|| { echo "first serve needs the Python 3.13 headers (apt install python3.13-dev)"; exit 1; }
|
|
[ -x "$CUDA_HOME/bin/nvcc" ] || { echo "no nvcc at CUDA_HOME=$CUDA_HOME"; exit 1; }
|
|
[ -e "$CUDA_HOME/lib64/libcudart.so" ] || { echo "$CUDA_HOME/lib64/libcudart.so missing (./build.sh creates it)"; exit 1; }
|
|
nv_rel="$("$CUDA_HOME/bin/nvcc" --version | sed -n 's/.*release \([0-9][0-9]*\.[0-9][0-9]*\),.*/\1/p' || true)"
|
|
[ -n "$nv_rel" ] || { echo "no CUDA release from \`$CUDA_HOME/bin/nvcc --version\`"; exit 1; }
|
|
drv_rel="$(nvidia-smi 2>/dev/null | sed -n 's/.*CUDA \(UMD \)\{0,1\}Version: *\([0-9][0-9]*\.[0-9][0-9]*\).*/\2/p' | head -1 || true)"
|
|
[ -n "$drv_rel" ] || echo "note: driver CUDA version unreadable; the nvcc-vs-driver check is skipped"
|
|
if [ -n "$drv_rel" ] && [ "$(printf '%s\n%s\n' "$nv_rel" "$drv_rel" | sort -V | tail -1)" != "$drv_rel" ]; then
|
|
echo "nvcc CUDA $nv_rel is newer than the driver's CUDA $drv_rel: first-serve kernels would be rejected."; exit 1
|
|
fi
|
|
|
|
# --- 5. topology (opt-in; default = the fixed shipped settings) ----------------------------------
|
|
case "${AUTO_TOPO:-0}" in
|
|
0) topo_on=0 ;;
|
|
1 | strict) topo_on=1 ;;
|
|
*) echo "topo: note: AUTO_TOPO='$AUTO_TOPO' is not 0, 1 or strict; the selector stays off"; topo_on=0 ;;
|
|
esac
|
|
topo_launch_prefix=""
|
|
legacy_topo_env() {
|
|
if [ "${1:-}" = fallback ]; then
|
|
if [ "${TOPO_CALLER_SET_CUDA_VISIBLE_DEVICES:-0}" = 1 ]; then CUDA_VISIBLE_DEVICES="$TOPO_CALLER_CUDA_VISIBLE_DEVICES"
|
|
else unset CUDA_VISIBLE_DEVICES; fi
|
|
case "${CUDA_VISIBLE_DEVICES:-}" in *[![:space:]]*) ;; *) unset CUDA_VISIBLE_DEVICES ;; esac
|
|
export TOPO_LAUNCH_PREFIX="" TOPO_DECISION_JSON=""; topo_launch_prefix=""
|
|
fi
|
|
export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3}"
|
|
export CUDA_DEVICE_ORDER=PCI_BUS_ID
|
|
export VLLM_SKIP_P2P_CHECK=1 NCCL_P2P_LEVEL=SYS NCCL_PROTO=LL
|
|
}
|
|
topo_env_file_ok() {
|
|
local f="$1" i=0 line
|
|
local val="([A-Za-z0-9_@%+=:,./-]+|'[^']*'(\"'\"'[^']*')*)"
|
|
local -a want=(
|
|
"^# topo_select v1 status=(ok|legacy) reason=[A-Za-z0-9_.-]+$"
|
|
"^export CUDA_VISIBLE_DEVICES=$val$"
|
|
"^export CUDA_DEVICE_ORDER=$val$"
|
|
"^export VLLM_SKIP_P2P_CHECK=$val$"
|
|
"^(export NCCL_P2P_LEVEL=$val|unset NCCL_P2P_LEVEL)$"
|
|
"^(export NCCL_PROTO=$val|unset NCCL_PROTO)$"
|
|
"^export TOPO_LAUNCH_PREFIX=('numactl --physcpubind=[0-9,-]+ --preferred=[0-9]+ --'|'')$"
|
|
"^export TOPO_DECISION_JSON=$val$")
|
|
[ -f "$f" ] || return 1
|
|
while IFS= read -r line || [ -n "$line" ]; do
|
|
[ "$i" -lt "${#want[@]}" ] || return 1
|
|
[[ $line =~ ${want[$i]} ]] || return 1
|
|
i=$((i + 1))
|
|
done < "$f" || return 1
|
|
[ "$i" -eq "${#want[@]}" ]
|
|
}
|
|
topo_unavailable() {
|
|
if [ "$AUTO_TOPO" = strict ]; then echo "topo: $1; AUTO_TOPO=strict refuses to guess"; exit 1; fi
|
|
echo "topo: $1, using legacy settings"; legacy_topo_env fallback
|
|
}
|
|
if [ "$topo_on" = 0 ]; then
|
|
legacy_topo_env
|
|
else
|
|
for v in CUDA_VISIBLE_DEVICES CUDA_DEVICE_ORDER VLLM_SKIP_P2P_CHECK NCCL_P2P_LEVEL NCCL_PROTO PLE_HOME; do
|
|
if [ "${!v+set}" = set ]; then export "TOPO_CALLER_SET_$v=1" "TOPO_CALLER_$v=${!v}"
|
|
else export "TOPO_CALLER_SET_$v=0"; fi
|
|
done
|
|
export TOPO_LAUNCH_PREFIX="" TOPO_DECISION_JSON=""
|
|
topo_timeout=20
|
|
case "${TOPO_TIMEOUT:-}" in
|
|
[1-9]|[1-9][0-9]|[1-9][0-9][0-9]) [ "$TOPO_TIMEOUT" -le 600 ] && topo_timeout="$TOPO_TIMEOUT" ;;
|
|
esac
|
|
topo_json_dir="${CACHE_ROOT:-$HERE/.vllm-cache}/topology"
|
|
tmp_topo_env=""
|
|
if tmp_topo_env="$(mktemp 2>/dev/null)" && [ -n "$tmp_topo_env" ]; then
|
|
trap 'rm -f -- "$tmp_topo_env" 2>/dev/null || :' EXIT
|
|
rc=0
|
|
TOPO_TIMEOUT="$topo_timeout" timeout --kill-after=5 "$((topo_timeout + 5))" \
|
|
"$VENV/bin/python" "$HERE/scripts/topo_select.py" --env-out "$tmp_topo_env" --json-dir "$topo_json_dir" || rc=$?
|
|
if [ "$rc" = 3 ]; then exit 1
|
|
elif [ "$rc" != 0 ]; then topo_unavailable "selector unavailable (rc=$rc)"
|
|
elif ! topo_env_file_ok "$tmp_topo_env"; then topo_unavailable "selector handed back an unusable env file"
|
|
else
|
|
# shellcheck disable=SC1090
|
|
if ! source "$tmp_topo_env"; then topo_unavailable "could not apply the selector's env file"
|
|
else topo_launch_prefix="$TOPO_LAUNCH_PREFIX"; fi
|
|
fi
|
|
rm -f -- "$tmp_topo_env" 2>/dev/null || :; trap - EXIT
|
|
else
|
|
topo_unavailable "could not create a temporary file"
|
|
fi
|
|
fi
|
|
if [ "${SERVE_PRINT_ENV_AND_EXIT:-0}" = 1 ]; then
|
|
echo "CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-} CUDA_DEVICE_ORDER=${CUDA_DEVICE_ORDER:-} VLLM_SKIP_P2P_CHECK=${VLLM_SKIP_P2P_CHECK:-} NCCL_P2P_LEVEL=${NCCL_P2P_LEVEL-<unset>} NCCL_PROTO=${NCCL_PROTO-<unset>} PREFIX=$topo_launch_prefix"
|
|
exit 0
|
|
fi
|
|
# --- end topology --------------------------------------------------------------------------------
|
|
|
|
# --- 6. the serve profile -------------------------------------------------------------------------
|
|
export VLLM_PP_LAYER_PARTITION=25,23
|
|
# the shipped profile sets every tree knob explicitly; drop inherited ones first
|
|
for v in $(env | sed -n 's/^\(MERLIN_[A-Z0-9_]*\|VLLM_E2_[A-Z0-9_]*\)=.*/\1/p'); do unset "$v"; done
|
|
unset VLLM_PLE_CPU_OFFLOAD VLLM_PLE_FP8_GLOBAL_SCALE VLLM_PLE_TEST_FAULT VLLM_PLE_TEST_FAULT_GATE VLLM_PLE_TEST_FAULT_LOG \
|
|
VLLM_QSA_KV_COLLECT VLLM_QSA_KV_CLIP_COUNT VLLM_E2_ALLOW_BREAKABLE VLLM_E2_EXPECTED_CUDAGRAPH_MODE \
|
|
VLLM_E2_MAMBA_SLOT_OWNERSHIP VLLM_E2_MEMSTATS VLLM_E2_MEMSTATS_DIR VLLM_E2_PLE_FLAG_CHECK \
|
|
VLLM_E2_SCHED_LOG VLLM_E2_ZERO_OWNERSHIP_CHECK VLLM_E2_GUARD_INTERVAL \
|
|
VLLM_PLE_OFFLOAD_WAIT_TIMEOUT VLLM_PLE_PROGRESS_DEADLINE_S VLLM_PLE_STARTUP_DEADLINE_S \
|
|
PYTORCH_CUDA_ALLOC_CONF || true
|
|
export VLLM_E2_PLE_PULL_TRANSPORT=1 VLLM_PLE_OFFLOAD_READY_TIMEOUT=1800
|
|
export VLLM_PLE_OFFLOAD_HOME_DEVICE="${PLE_HOME:-0}"
|
|
export VLLM_HOST_EMBED_TABLE=1
|
|
export VLLM_USE_BREAKABLE_CUDAGRAPH=0
|
|
export VLLM_QSA_KV_SCALES="${SCALES:-$CKPT/qsa_kv_scales.json}"
|
|
# Engine stats line (throughput, KV usage, queue). 1 s matches how often the numbers are useful.
|
|
# Unset, vLLM waits 10 s (vllm/envs.py:47), so a quiet run looks idle for ten seconds.
|
|
export VLLM_LOG_STATS_INTERVAL="${VLLM_LOG_STATS_INTERVAL:-1}"
|
|
export HF_HUB_OFFLINE=1 HF_DATASETS_OFFLINE=1
|
|
# the kernel toggles (the shipped values). These names are the tree's own: envs.py in the
|
|
# serving tree registers MERLIN_* for the kernel toggles and VLLM_E2_* for the runtime knobs.
|
|
export MERLIN_FULL_K=1 MERLIN_QSA_SORT=1 MERLIN_TIE_DET=1 MERLIN_TIE_RECENT=1 MERLIN_TOPK_SORTED_EMIT=1
|
|
export VLLM_E2_GUARD_MODE="${GUARD:-warn}"
|
|
export VLLM_E2_SIDE_CACHE_LAYOUT=1 VLLM_USE_V2_MODEL_RUNNER=1 VLLM_E2_MAMBA_RETAIN_CHECKPOINT=1
|
|
# Accept any name in the model field of a request, like llama.cpp. Upstream's own switch, on by
|
|
# default here: /v1/models and the model field of every reply still report the served name.
|
|
# Set VLLM_SKIP_MODEL_NAME_VALIDATION=0 to reject a name that is not served.
|
|
export VLLM_SKIP_MODEL_NAME_VALIDATION="${VLLM_SKIP_MODEL_NAME_VALIDATION:-1}"
|
|
if [ "${COUNTERS:-0}" = 1 ]; then export MERLIN_TOPK_EXACT_COUNTERS_LOG=30 VLLM_QSA_KV_CLIP_COUNT=300; fi
|
|
[ -f "$VLLM_QSA_KV_SCALES" ] || { echo "scales sidecar missing: $VLLM_QSA_KV_SCALES (FP8 KV would run at scale 1.0). Set SCALES=<path> to the sidecar of this model."; exit 1; }
|
|
|
|
# Concurrency and the prefill chunk, tunable from .env. vLLM refuses a chunk smaller than the
|
|
# sequence count (vllm/config/scheduler.py:309); say it here, where the reader set the value.
|
|
MAX_SEQS="${MAX_SEQS:-4}"
|
|
MAX_BATCHED_TOKENS="${MAX_BATCHED_TOKENS:-2048}"
|
|
for v in MAX_SEQS MAX_BATCHED_TOKENS; do
|
|
case "${!v}" in ''|*[!0-9]*) echo "$v must be a positive integer (got: ${!v})"; exit 1 ;; esac
|
|
[ "${!v}" -ge 1 ] || { echo "$v must be 1 or more (got: ${!v})"; exit 1; }
|
|
done
|
|
if [ "$MAX_BATCHED_TOKENS" -lt "$MAX_SEQS" ]; then
|
|
echo "MAX_BATCHED_TOKENS ($MAX_BATCHED_TOKENS) must be >= MAX_SEQS ($MAX_SEQS)"; exit 1
|
|
fi
|
|
# The shipped pool holds 924,993 tokens, pinned by --kv-cache-memory. A shape that needs more does
|
|
# not fail: the engine preempts and recomputes.
|
|
if [ $((MAX_SEQS * 262144)) -gt 924993 ]; then
|
|
echo "note: MAX_SEQS=$MAX_SEQS wants $((MAX_SEQS * 262144)) tokens at the full 262144 context; the pool holds 924993, so full-length requests preempt"
|
|
fi
|
|
# CUDA graphs must cover the widest decode batch: MAX_SEQS x (K+1) tokens, K the MTP depth. A
|
|
# batch pads up to the next captured size, and runs outside the graph above the largest one. The
|
|
# sizes below the MTP width stay for small and mixed batches. Each size costs capture time and
|
|
# VRAM, so the list stops at the widest batch, as the shipped shape does (4 x 4 = 16).
|
|
# MTP depth: 3 is Qwen 3.8's recommended value, not a knob. The graph math below reads it, so keep
|
|
# it equal to the num_speculative_tokens in --speculative-config.
|
|
MTP_K=3
|
|
CG_TOP=$((MAX_SEQS * (MTP_K + 1)))
|
|
CG_SIZES="1,2,4"
|
|
for ((n = MTP_K + 1; n <= CG_TOP; n += MTP_K + 1)); do
|
|
[ "$n" -gt 4 ] && CG_SIZES="$CG_SIZES,$n"
|
|
done
|
|
CG_COUNT=$((2 + MAX_SEQS))
|
|
if [ "$CG_COUNT" -gt 8 ]; then
|
|
echo "note: $CG_COUNT CUDA graphs for MAX_SEQS=$MAX_SEQS; capture time and graph VRAM both grow with the count"
|
|
fi
|
|
echo "cuda graphs: 1,2,4 then every $((MTP_K + 1)) tokens to $CG_TOP"
|
|
|
|
# Compile cache: on disk by default, so it survives a reboot. A cold cache costs ~6 min of kernel
|
|
# compile, and the cache is only ~160 MB, so disk is the right home for it. Keep ONE root per
|
|
# deployment -- a new root costs a fresh compile. CACHE_ROOT=/vllm-cache (a tmpfs) is the opt-in
|
|
# for a box where you want it in RAM.
|
|
[ -d "$CACHE_ROOT" ] || { mkdir -p "$CACHE_ROOT" 2>/dev/null || { echo "cannot create $CACHE_ROOT"; exit 1; }; }
|
|
if df -T "$CACHE_ROOT" 2>/dev/null | grep -q tmpfs; then
|
|
echo "note: $CACHE_ROOT is a tmpfs; the compile cache is lost on reboot"
|
|
fi
|
|
export VLLM_CACHE_ROOT="$CACHE_ROOT"
|
|
export FLASHINFER_WORKSPACE_BASE="$CACHE_ROOT/flashinfer" TRITON_CACHE_DIR="$CACHE_ROOT/triton"
|
|
export PYTHONPATH="$TREE" VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0
|
|
|
|
# host RAM (the n-gram table lives here; a short host OOMs mid-boot) and /dev/shm
|
|
mem_kb="$(awk '/^MemTotal:/{print $2}' /proc/meminfo 2>/dev/null || true)"
|
|
for f in /sys/fs/cgroup/memory.max /sys/fs/cgroup/memory/memory.limit_in_bytes; do
|
|
lim="$(cat "$f" 2>/dev/null || true)"
|
|
case "$lim" in ''|*[!0-9]*) continue ;; esac
|
|
lim_kb=$((lim / 1024)); if [ -z "${mem_kb:-}" ] || [ "$lim_kb" -lt "$mem_kb" ]; then mem_kb="$lim_kb"; fi
|
|
done
|
|
if [ -n "${mem_kb:-}" ] && [ "$mem_kb" -lt 100663296 ]; then
|
|
echo "WARNING: ${mem_kb} KB visible RAM (< the required 96 GiB); resident floor is ~69 GiB"
|
|
fi
|
|
shm_kb="$(df -Pk /dev/shm 2>/dev/null | awk 'NR==2{print $2}' || true)"
|
|
case "$shm_kb" in *[!0-9]*) shm_kb= ;; esac
|
|
if [ -n "${shm_kb:-}" ] && [ "$shm_kb" -lt 1048576 ]; then
|
|
echo "WARNING: /dev/shm is $((shm_kb / 1024)) MB; the PLE transport needs >= 1 GB"
|
|
fi
|
|
|
|
read -r -a NAMES <<< "${MODEL_NAME:-qwen38-flash-next}"
|
|
echo "serving $CKPT as ${NAMES[*]} on ${HOST:-127.0.0.1}:${PORT:-8000} (cache $CACHE_ROOT)"
|
|
if [ -n "$LOG" ]; then
|
|
# stdout + stderr, so the PLE offload child and the engine workers land in the same file.
|
|
# systemd/vllm.logrotate rotates it: daily, 100 MB cap, 14 files, gzip. Read it with: tail -f $LOG
|
|
touch "$LOG" 2>/dev/null || { echo "cannot write $LOG; run as root or set LOG=<path>"; exit 1; }
|
|
echo "log: $LOG (rotate: systemd/vllm.logrotate)"
|
|
exec >> "$LOG" 2>&1
|
|
fi
|
|
|
|
# unquoted on purpose: word-splits into a numactl prefix from the validated selector file, empty when off
|
|
exec ${topo_launch_prefix:+$topo_launch_prefix} "$VENV/bin/vllm" serve "$CKPT" \
|
|
--served-model-name "${NAMES[@]}" \
|
|
--host "${HOST:-127.0.0.1}" --port "${PORT:-8000}" \
|
|
--tensor-parallel-size 2 --pipeline-parallel-size 2 --enable-expert-parallel \
|
|
--speculative-config '{"method":"mtp","num_speculative_tokens":3,"draft_sample_method":"probabilistic"}' \
|
|
--max-model-len 262144 --max-num-seqs "$MAX_SEQS" --max-num-batched-tokens "$MAX_BATCHED_TOKENS" \
|
|
--gpu-memory-utilization 0.96 --kv-cache-memory 4100000000 --kv-cache-dtype fp8_e4m3 \
|
|
--offload-towers --limit-mm-per-prompt '{"image":42,"video":0}' \
|
|
--mm-processor-kwargs '{"size":{"longest_edge":4194304,"shortest_edge":65536}}' \
|
|
--compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":['"$CG_SIZES"']}' \
|
|
--enable-prefix-caching --enable-prompt-tokens-details \
|
|
--no-enable-flashinfer-autotune \
|
|
--enable-auto-tool-choice --tool-call-parser qwen3_coder --reasoning-parser qwen3 \
|
|
--override-generation-config '{"temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0}' \
|
|
--default-chat-template-kwargs '{"enable_thinking": true, "reasoning_effort": "xhigh"}' \
|
|
--shutdown-timeout 10 \
|
|
--block-size 4096 --use-replayssm \
|
|
--prefix-match-unit 64 \
|
|
--disable-uvicorn-access-log \
|
|
"$@"
|