#!/usr/bin/env bash # Serve Qwen3.8-Flash-Next-W4A16 on 4x RTX 3090. Builds first when the tree or the venv is # missing, checks the box, then runs the engine. One script for bare metal and for the image. # # sudo ./start.sh [checkpoint-dir] [extra vllm args] # build on first run, then serve # ./build.sh --force # compile again (see ./build.sh) # # Shape: TP2 x PP2 + EP, MTP K=3, 262K per request, ~925K-token FP8 KV pool, full 262K context, # host-resident PLE/embeddings, prefix caching with a 64-token match unit, RecoverSSM, side-cache # layout. This tree's profile guard boots ONLY the command at the end of this script; # --enforce-eager, other cudagraph modes and breakable graphs are refused. # # Env: # CKPT / MODEL_MOUNT_PATH checkpoint dir (bare metal: the Swift1.5 Qwen3.8-Flash-Next W4A16 # build; container: /models/Qwen3.8-Flash-Next). A first argument that starts with # `-` is a vllm argument, so the default checkpoint is kept. # TREE VENV serving tree and venv (default: ./vllm and ./.venv, built by build.sh) # PORT (8000) HOST (127.0.0.1) MODEL_NAME ("name1 name2 ...", first is the health probe's) # SCALES KV scale sidecar (default: qsa_kv_scales_swift.json inside the checkpoint) # CACHE_ROOT compile/cudagraph cache (default: .vllm-cache next to this script, on disk) # PLE_HOME GPU that hosts the PLE offload home (0) # GUARD bounds-guard telemetry off|warn|raise (warn) # COUNTERS=1 logging-only counters (top-k exact restarts /30s, FP8 KV clips /300s) # VLLM_LOG_STATS_INTERVAL seconds between engine stats lines (unset = vLLM default 1 s) # CUDA_HOME toolkit for first-serve JIT; default: the venv's CUDA 13.0 wheels # AUTO_TOPO 0 (default, fixed shipped settings) | 1 | strict -> scripts/topo_select.py # AUTO_TOPO_NCCL=default AUTO_TOPO_BIND=1 LANE_GPUS=auto LANE_CPUS=... TOPO_TIMEOUT=20 # LOG log file (/var/log/vllm.log). LOG= (empty) keeps the output on the terminal, # which is what the image does so `docker compose logs` works. # # First serve compiles kernels (~6 min, writes $CACHE_ROOT). RAM: the PLE n-gram table is host # memory, resident floor ~69 GiB. /dev/shm >= 1 GB. One CUDA process per GPU. set -euo pipefail HERE="$(cd "$(dirname "$0")" && pwd)" TREE="${TREE:-$HERE/vllm}"; VENV="${VENV:-$HERE/.venv}" CKPT="${CKPT:-${MODEL_MOUNT_PATH:-/data/models/halt95/Swift1.5-Qwen3.8-Flash-Next-W4A16-Merlin}}" CACHE_ROOT="${CACHE_ROOT:-$HERE/.vllm-cache}" # Colon-free default: LOG= (set, empty) is the container's "do not redirect". LOG="${LOG-/var/log/vllm.log}" if [ $# -gt 0 ]; then case "$1" in -*) ;; "") shift ;; *) CKPT="$1"; shift ;; esac fi case "$TREE" in /*) ;; *) TREE="$PWD/$TREE" ;; esac case "$VENV" in /*) ;; *) VENV="$PWD/$VENV" ;; esac # --- 1. build when this box has no serving tree or no venv --------------------------------------- mkdir -p "$CACHE_ROOT" 2>/dev/null || { echo "cannot create $CACHE_ROOT; set CACHE_ROOT="; exit 1; } if mountpoint -q "$CACHE_ROOT" 2>/dev/null; then cache_kind="RAM"; else cache_kind="disk"; fi echo "cache: $CACHE_ROOT ($cache_kind)" if [ ! -f "$TREE/vllm/__init__.py" ] || [ ! -x "$VENV/bin/vllm" ]; then echo "first run: building the tree and the venv (./build.sh, ~20 min on 64 cores)" "$HERE/build.sh" fi # --- 2. the checkpoint --------------------------------------------------------------------------- if [ ! -f "$CKPT/config.json" ]; then echo "checkpoint not found at $CKPT" echo "bare metal: sudo ./start.sh /path/to/Qwen3.8-Flash-Next-W4A16-Merlin" echo "container: -v /path/to/Qwen3.8-Flash-Next-W4A16-Merlin:/models/Qwen3.8-Flash-Next:ro" exit 1 fi if ! "$VENV/bin/python" -c 'import json,sys; c=json.load(open(sys.argv[1])); sys.exit(0 if c.get("text_config",{}).get("ple_embedding_dtype")=="float8_e4m3fn" else 1)' "$CKPT/config.json"; then if [ -w "$CKPT" ] && [ -w "$CKPT/config.json" ]; then echo "config.json lacks text_config.ple_embedding_dtype; adding it (weights untouched, backup kept)" python3 "$HERE/scripts/make-e1-config.py" "$CKPT" else echo "config.json lacks text_config.ple_embedding_dtype and the directory is read-only." echo "Run once on the writable copy: python3 scripts/make-e1-config.py $CKPT" exit 1 fi fi if ! command -v nvidia-smi >/dev/null 2>&1 || ! nvidia-smi -L >/dev/null 2>&1; then echo "WARNING: no NVIDIA GPU visible (in Docker: --gpus all, and nvidia-ctk runtime configure --runtime=docker)" fi # --- 3. the build products ----------------------------------------------------------------------- [ -f "$TREE/vllm/__init__.py" ] || { echo "TREE=$TREE is not the serving tree"; exit 1; } [ -f "$TREE/.build-products" ] && [ -s "$TREE/.build-products.sha256" ] \ || { echo "TREE=$TREE has no verified build products (./build.sh)"; exit 1; } (cd "$TREE" && sha256sum -c --quiet --strict .build-products.sha256) \ || { echo "TREE=$TREE: build products changed since ./build.sh verified them"; exit 1; } [ -x "$VENV/bin/python" ] || { echo "VENV=$VENV has no bin/python"; exit 1; } # A vllm/ in the launch directory shadows $TREE: `python -m` puts CWD before PYTHONPATH. if [ -f "$PWD/vllm/__init__.py" ] && [ "$(cd "$PWD/vllm" 2>/dev/null && pwd -P)" != "$(cd "$TREE/vllm" && pwd -P)" ]; then echo "refusing to start: $PWD/vllm would shadow $TREE/vllm; start from another directory"; exit 1 fi # --- 4. first-serve toolchain (FlashInfer JITs against CUDA_HOME, Triton against Python.h) ------- if [ -z "${CUDA_HOME:-}" ]; then CUDA_HOME="$("$VENV/bin/python" -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')/nvidia/cu13" fi export CUDA_HOME PATH="$CUDA_HOME/bin:$PATH" for t in cc c++ ninja; do command -v "$t" >/dev/null || { echo "first serve needs $t (apt install gcc g++ ninja-build)"; exit 1; }; done "$VENV/bin/python" -c 'import os,sys,sysconfig; sys.exit(0 if os.path.exists(os.path.join(sysconfig.get_paths()["include"],"Python.h")) else 1)' \ || { echo "first serve needs the Python 3.13 headers (apt install python3.13-dev)"; exit 1; } [ -x "$CUDA_HOME/bin/nvcc" ] || { echo "no nvcc at CUDA_HOME=$CUDA_HOME"; exit 1; } [ -e "$CUDA_HOME/lib64/libcudart.so" ] || { echo "$CUDA_HOME/lib64/libcudart.so missing (./build.sh creates it)"; exit 1; } nv_rel="$("$CUDA_HOME/bin/nvcc" --version | sed -n 's/.*release \([0-9][0-9]*\.[0-9][0-9]*\),.*/\1/p' || true)" [ -n "$nv_rel" ] || { echo "no CUDA release from \`$CUDA_HOME/bin/nvcc --version\`"; exit 1; } drv_rel="$(nvidia-smi 2>/dev/null | sed -n 's/.*CUDA \(UMD \)\{0,1\}Version: *\([0-9][0-9]*\.[0-9][0-9]*\).*/\2/p' | head -1 || true)" [ -n "$drv_rel" ] || echo "note: driver CUDA version unreadable; the nvcc-vs-driver check is skipped" if [ -n "$drv_rel" ] && [ "$(printf '%s\n%s\n' "$nv_rel" "$drv_rel" | sort -V | tail -1)" != "$drv_rel" ]; then echo "nvcc CUDA $nv_rel is newer than the driver's CUDA $drv_rel: first-serve kernels would be rejected."; exit 1 fi # --- 5. topology (opt-in; default = the fixed shipped settings) ---------------------------------- case "${AUTO_TOPO:-0}" in 0) topo_on=0 ;; 1 | strict) topo_on=1 ;; *) echo "topo: note: AUTO_TOPO='$AUTO_TOPO' is not 0, 1 or strict; the selector stays off"; topo_on=0 ;; esac topo_launch_prefix="" legacy_topo_env() { if [ "${1:-}" = fallback ]; then if [ "${TOPO_CALLER_SET_CUDA_VISIBLE_DEVICES:-0}" = 1 ]; then CUDA_VISIBLE_DEVICES="$TOPO_CALLER_CUDA_VISIBLE_DEVICES" else unset CUDA_VISIBLE_DEVICES; fi case "${CUDA_VISIBLE_DEVICES:-}" in *[![:space:]]*) ;; *) unset CUDA_VISIBLE_DEVICES ;; esac export TOPO_LAUNCH_PREFIX="" TOPO_DECISION_JSON=""; topo_launch_prefix="" fi export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3}" export CUDA_DEVICE_ORDER=PCI_BUS_ID export VLLM_SKIP_P2P_CHECK=1 NCCL_P2P_LEVEL=SYS NCCL_PROTO=LL } topo_env_file_ok() { local f="$1" i=0 line local val="([A-Za-z0-9_@%+=:,./-]+|'[^']*'(\"'\"'[^']*')*)" local -a want=( "^# topo_select v1 status=(ok|legacy) reason=[A-Za-z0-9_.-]+$" "^export CUDA_VISIBLE_DEVICES=$val$" "^export CUDA_DEVICE_ORDER=$val$" "^export VLLM_SKIP_P2P_CHECK=$val$" "^(export NCCL_P2P_LEVEL=$val|unset NCCL_P2P_LEVEL)$" "^(export NCCL_PROTO=$val|unset NCCL_PROTO)$" "^export TOPO_LAUNCH_PREFIX=('numactl --physcpubind=[0-9,-]+ --preferred=[0-9]+ --'|'')$" "^export TOPO_DECISION_JSON=$val$") [ -f "$f" ] || return 1 while IFS= read -r line || [ -n "$line" ]; do [ "$i" -lt "${#want[@]}" ] || return 1 [[ $line =~ ${want[$i]} ]] || return 1 i=$((i + 1)) done < "$f" || return 1 [ "$i" -eq "${#want[@]}" ] } topo_unavailable() { if [ "$AUTO_TOPO" = strict ]; then echo "topo: $1; AUTO_TOPO=strict refuses to guess"; exit 1; fi echo "topo: $1, using legacy settings"; legacy_topo_env fallback } if [ "$topo_on" = 0 ]; then legacy_topo_env else for v in CUDA_VISIBLE_DEVICES CUDA_DEVICE_ORDER VLLM_SKIP_P2P_CHECK NCCL_P2P_LEVEL NCCL_PROTO PLE_HOME; do if [ "${!v+set}" = set ]; then export "TOPO_CALLER_SET_$v=1" "TOPO_CALLER_$v=${!v}" else export "TOPO_CALLER_SET_$v=0"; fi done export TOPO_LAUNCH_PREFIX="" TOPO_DECISION_JSON="" topo_timeout=20 case "${TOPO_TIMEOUT:-}" in [1-9]|[1-9][0-9]|[1-9][0-9][0-9]) [ "$TOPO_TIMEOUT" -le 600 ] && topo_timeout="$TOPO_TIMEOUT" ;; esac topo_json_dir="${CACHE_ROOT:-$HERE/.vllm-cache}/topology" tmp_topo_env="" if tmp_topo_env="$(mktemp 2>/dev/null)" && [ -n "$tmp_topo_env" ]; then trap 'rm -f -- "$tmp_topo_env" 2>/dev/null || :' EXIT rc=0 TOPO_TIMEOUT="$topo_timeout" timeout --kill-after=5 "$((topo_timeout + 5))" \ "$VENV/bin/python" "$HERE/scripts/topo_select.py" --env-out "$tmp_topo_env" --json-dir "$topo_json_dir" || rc=$? if [ "$rc" = 3 ]; then exit 1 elif [ "$rc" != 0 ]; then topo_unavailable "selector unavailable (rc=$rc)" elif ! topo_env_file_ok "$tmp_topo_env"; then topo_unavailable "selector handed back an unusable env file" else # shellcheck disable=SC1090 if ! source "$tmp_topo_env"; then topo_unavailable "could not apply the selector's env file" else topo_launch_prefix="$TOPO_LAUNCH_PREFIX"; fi fi rm -f -- "$tmp_topo_env" 2>/dev/null || :; trap - EXIT else topo_unavailable "could not create a temporary file" fi fi if [ "${SERVE_PRINT_ENV_AND_EXIT:-0}" = 1 ]; then echo "CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-} CUDA_DEVICE_ORDER=${CUDA_DEVICE_ORDER:-} VLLM_SKIP_P2P_CHECK=${VLLM_SKIP_P2P_CHECK:-} NCCL_P2P_LEVEL=${NCCL_P2P_LEVEL-} NCCL_PROTO=${NCCL_PROTO-} PREFIX=$topo_launch_prefix" exit 0 fi # --- end topology -------------------------------------------------------------------------------- # --- 6. the serve profile ------------------------------------------------------------------------- export VLLM_PP_LAYER_PARTITION=25,23 # the shipped profile sets every tree knob explicitly; drop inherited ones first for v in $(env | sed -n 's/^\(MERLIN_[A-Z0-9_]*\|VLLM_E2_[A-Z0-9_]*\)=.*/\1/p'); do unset "$v"; done unset VLLM_PLE_CPU_OFFLOAD VLLM_PLE_FP8_GLOBAL_SCALE VLLM_PLE_TEST_FAULT VLLM_PLE_TEST_FAULT_GATE VLLM_PLE_TEST_FAULT_LOG \ VLLM_QSA_KV_COLLECT VLLM_QSA_KV_CLIP_COUNT VLLM_E2_ALLOW_BREAKABLE VLLM_E2_EXPECTED_CUDAGRAPH_MODE \ VLLM_E2_MAMBA_SLOT_OWNERSHIP VLLM_E2_MEMSTATS VLLM_E2_MEMSTATS_DIR VLLM_E2_PLE_FLAG_CHECK \ VLLM_E2_SCHED_LOG VLLM_E2_ZERO_OWNERSHIP_CHECK VLLM_E2_GUARD_INTERVAL \ VLLM_PLE_OFFLOAD_WAIT_TIMEOUT VLLM_PLE_PROGRESS_DEADLINE_S VLLM_PLE_STARTUP_DEADLINE_S \ PYTORCH_CUDA_ALLOC_CONF || true export VLLM_E2_PLE_PULL_TRANSPORT=1 VLLM_PLE_OFFLOAD_READY_TIMEOUT=1800 export VLLM_PLE_OFFLOAD_HOME_DEVICE="${PLE_HOME:-0}" export VLLM_HOST_EMBED_TABLE=1 export VLLM_USE_BREAKABLE_CUDAGRAPH=0 export VLLM_QSA_KV_SCALES="${SCALES:-$CKPT/qsa_kv_scales_swift.json}" # engine stats line (throughput, KV usage, queue). Set empty to keep vLLM's default. if [ -n "${VLLM_LOG_STATS_INTERVAL:-}" ]; then export VLLM_LOG_STATS_INTERVAL; fi export HF_HUB_OFFLINE=1 HF_DATASETS_OFFLINE=1 # the kernel toggles (the shipped values). These names are the tree's own: envs.py in the # serving tree registers MERLIN_* for the kernel toggles and VLLM_E2_* for the runtime knobs. export MERLIN_FULL_K=1 MERLIN_QSA_SORT=1 MERLIN_TIE_DET=1 MERLIN_TIE_RECENT=1 MERLIN_TOPK_SORTED_EMIT=1 export VLLM_E2_GUARD_MODE="${GUARD:-warn}" export VLLM_E2_SIDE_CACHE_LAYOUT=1 VLLM_USE_V2_MODEL_RUNNER=1 VLLM_E2_MAMBA_RETAIN_CHECKPOINT=1 if [ "${COUNTERS:-0}" = 1 ]; then export MERLIN_TOPK_EXACT_COUNTERS_LOG=30 VLLM_QSA_KV_CLIP_COUNT=300; fi [ -f "$VLLM_QSA_KV_SCALES" ] || { echo "scales sidecar missing: $VLLM_QSA_KV_SCALES (FP8 KV would run at scale 1.0)"; exit 1; } # Compile cache: on disk by default, so it survives a reboot. A cold cache costs ~6 min of kernel # compile, and the cache is only ~160 MB, so disk is the right home for it. Keep ONE root per # deployment -- a new root costs a fresh compile. CACHE_ROOT=/vllm-cache (a tmpfs) is the opt-in # for a box where you want it in RAM. [ -d "$CACHE_ROOT" ] || { mkdir -p "$CACHE_ROOT" 2>/dev/null || { echo "cannot create $CACHE_ROOT"; exit 1; }; } if df -T "$CACHE_ROOT" 2>/dev/null | grep -q tmpfs; then echo "note: $CACHE_ROOT is a tmpfs; the compile cache is lost on reboot" fi export VLLM_CACHE_ROOT="$CACHE_ROOT" export FLASHINFER_WORKSPACE_BASE="$CACHE_ROOT/flashinfer" TRITON_CACHE_DIR="$CACHE_ROOT/triton" export PYTHONPATH="$TREE" VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 # host RAM (the n-gram table lives here; a short host OOMs mid-boot) and /dev/shm mem_kb="$(awk '/^MemTotal:/{print $2}' /proc/meminfo 2>/dev/null || true)" for f in /sys/fs/cgroup/memory.max /sys/fs/cgroup/memory/memory.limit_in_bytes; do lim="$(cat "$f" 2>/dev/null || true)" case "$lim" in ''|*[!0-9]*) continue ;; esac lim_kb=$((lim / 1024)); if [ -z "${mem_kb:-}" ] || [ "$lim_kb" -lt "$mem_kb" ]; then mem_kb="$lim_kb"; fi done if [ -n "${mem_kb:-}" ] && [ "$mem_kb" -lt 100663296 ]; then echo "WARNING: ${mem_kb} KB visible RAM (< the required 96 GiB); resident floor is ~69 GiB" fi shm_kb="$(df -Pk /dev/shm 2>/dev/null | awk 'NR==2{print $2}' || true)" case "$shm_kb" in *[!0-9]*) shm_kb= ;; esac if [ -n "${shm_kb:-}" ] && [ "$shm_kb" -lt 1048576 ]; then echo "WARNING: /dev/shm is $((shm_kb / 1024)) MB; the PLE transport needs >= 1 GB" fi read -r -a NAMES <<< "${MODEL_NAME:-qwen38-flash-next}" echo "serving $CKPT as ${NAMES[*]} on ${HOST:-127.0.0.1}:${PORT:-8000} (cache $CACHE_ROOT)" if [ -n "$LOG" ]; then # stdout + stderr, so the PLE offload child and the engine workers land in the same file. # systemd/vllm.logrotate rotates it: daily, 100 MB cap, 14 files, gzip. Read it with: tail -f $LOG touch "$LOG" 2>/dev/null || { echo "cannot write $LOG; run as root or set LOG="; exit 1; } echo "log: $LOG (rotate: systemd/vllm.logrotate)" exec >> "$LOG" 2>&1 fi # unquoted on purpose: word-splits into a numactl prefix from the validated selector file, empty when off exec ${topo_launch_prefix:+$topo_launch_prefix} "$VENV/bin/vllm" serve "$CKPT" \ --served-model-name "${NAMES[@]}" \ --host "${HOST:-127.0.0.1}" --port "${PORT:-8000}" \ --tensor-parallel-size 2 --pipeline-parallel-size 2 --enable-expert-parallel \ --speculative-config '{"method":"mtp","num_speculative_tokens":3,"draft_sample_method":"probabilistic"}' \ --max-model-len 262144 --max-num-seqs 2 --max-num-batched-tokens 4096 \ --gpu-memory-utilization 0.96 --kv-cache-memory 4100000000 --kv-cache-dtype fp8_e4m3 \ --limit-mm-per-prompt '{"image":42,"video":0}' --mm-processor-cache-gb 0 \ --mm-processor-kwargs '{"size":{"longest_edge":4194304,"shortest_edge":65536}}' \ --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,6,8,12,16,24,32]}' \ --enable-prefix-caching --enable-prompt-tokens-details \ --no-enable-flashinfer-autotune \ --enable-auto-tool-choice --tool-call-parser qwen3_coder --reasoning-parser qwen3 \ --override-generation-config '{"temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0}' \ --default-chat-template-kwargs '{"enable_thinking": true, "reasoning_effort": "low"}' \ --shutdown-timeout 10 \ --block-size 4096 --use-replayssm \ --prefix-match-unit 64 \ --safetensors-load-strategy prefetch \ "$@"