vLLM-4x-RTX-3090/start.sh

#!/usr/bin/env bash

# Serve Qwen3.8-Flash-Next-W4A16 on 4x RTX 3090. Builds first when the tree or the venv is
# missing, checks the box, then runs the engine. One script for bare metal and for the image.
#
#   sudo ./start.sh [checkpoint-dir] [extra vllm args]     # build on first run, then serve
#   ./build.sh --force                                     # compile again (see ./build.sh)
#
# Shape: TP2 x PP2 + EP, MTP K=3, 262K per request, ~925K-token FP8 KV pool, full 262K context,
# host-resident PLE/embeddings, prefix caching with a 64-token match unit, RecoverSSM, side-cache
# layout. This tree's profile guard boots ONLY the command at the end of this script;
# --enforce-eager, other cudagraph modes and breakable graphs are refused.
#
# Env:
#   CKPT / MODEL_MOUNT_PATH  checkpoint dir (bare metal: the Swift1.5 Qwen3.8-Flash-Next W4A16
#                            build; container: /models/Qwen3.8-Flash-Next). A first argument that starts with
#                            `-` is a vllm argument, so the default checkpoint is kept.
#   TREE VENV                serving tree and venv (default: ./vllm and ./.venv, built by build.sh)
#   PORT (8000) HOST (127.0.0.1) MODEL_NAME ("name1 name2 ...", first is the health probe's)
#   SCALES      KV scale sidecar, host path (default: $CKPT/qsa_kv_scales.json)
#   CACHE_ROOT  compile/cudagraph cache (default: .vllm-cache next to this script, on disk)
#   PLE_HOME    GPU that hosts the PLE offload home (0)
#   GUARD       bounds-guard telemetry off|warn|raise (warn)
#   COUNTERS=1    logging-only counters (top-k exact restarts /30s, FP8 KV clips /300s)
#   VLLM_LOG_STATS_INTERVAL  seconds between engine stats lines (1; the vLLM default is 10 s)
#   MAX_SEQS (4)             concurrent requests. The CUDA graph list follows it, to MAX_SEQS x 4
#                            tokens (MTP K=3). N x 262144 tokens above the 924,993-token pool preempts.
#   MAX_BATCHED_TOKENS (2048) prefill chunk in tokens. Must be >= MAX_SEQS. 2048 matches
#                            --block-size; 4096 prefills one max-size image in one chunk.
#   CUDA_HOME   toolkit for first-serve JIT; default: the venv's CUDA 13.0 wheels
#   AUTO_TOPO   0 (default, fixed shipped settings) | 1 | strict -> scripts/topo_select.py
#   AUTO_TOPO_NCCL=default  AUTO_TOPO_BIND=1  LANE_GPUS=auto  LANE_CPUS=...  TOPO_TIMEOUT=20
#   LOG         log file (/var/log/vllm.log). LOG= (empty) keeps the output on the terminal,
#               which is what the image does so `docker compose logs` works.
#
# First serve compiles kernels (~6 min, writes $CACHE_ROOT). RAM: the PLE n-gram table is host
# memory, resident floor ~69 GiB. /dev/shm >= 1 GB. One CUDA process per GPU.

set -euo pipefail

HERE="$(cd "$(dirname "$0")" && pwd)"
TREE="${TREE:-$HERE/vllm}"; VENV="${VENV:-$HERE/.venv}"
CKPT="${CKPT:-${MODEL_MOUNT_PATH:-/data/models/halt95/Swift1.5-Qwen3.8-Flash-Next-W4A16-Merlin}}"
CACHE_ROOT="${CACHE_ROOT:-$HERE/.vllm-cache}"
# Colon-free default: LOG= (set, empty) is the container's "do not redirect".
LOG="${LOG-/var/log/vllm.log}"

if [ $# -gt 0 ]; then
  case "$1" in -*) ;; "") shift ;; *) CKPT="$1"; shift ;; esac
fi
case "$TREE" in /*) ;; *) TREE="$PWD/$TREE" ;; esac
case "$VENV" in /*) ;; *) VENV="$PWD/$VENV" ;; esac

# --- 1. build when this box has no serving tree or no venv ---------------------------------------
mkdir -p "$CACHE_ROOT" 2>/dev/null || { echo "cannot create $CACHE_ROOT; set CACHE_ROOT=<path>"; exit 1; }
if mountpoint -q "$CACHE_ROOT" 2>/dev/null; then cache_kind="RAM"; else cache_kind="disk"; fi
echo "cache: $CACHE_ROOT ($cache_kind)"
if [ ! -f "$TREE/vllm/__init__.py" ] || [ ! -x "$VENV/bin/vllm" ]; then
  echo "first run: building the tree and the venv (./build.sh, ~20 min on 64 cores)"
  "$HERE/build.sh"
fi

# --- 2. the checkpoint ---------------------------------------------------------------------------
if [ ! -f "$CKPT/config.json" ]; then
  echo "checkpoint not found at $CKPT"
  echo "bare metal:  sudo ./start.sh /path/to/Qwen3.8-Flash-Next-W4A16-Merlin"
  echo "container:   -v /path/to/Qwen3.8-Flash-Next-W4A16-Merlin:/models/Qwen3.8-Flash-Next:ro"
  exit 1
fi
if ! "$VENV/bin/python" -c 'import json,sys; c=json.load(open(sys.argv[1])); sys.exit(0 if c.get("text_config",{}).get("ple_embedding_dtype")=="float8_e4m3fn" else 1)' "$CKPT/config.json"; then
  if [ -w "$CKPT" ] && [ -w "$CKPT/config.json" ]; then
    echo "config.json lacks text_config.ple_embedding_dtype; adding it (weights untouched, backup kept)"
    python3 "$HERE/scripts/make-e1-config.py" "$CKPT"
  else
    echo "config.json lacks text_config.ple_embedding_dtype and the directory is read-only."
    echo "Run once on the writable copy: python3 scripts/make-e1-config.py $CKPT"
    exit 1
  fi
fi
if ! command -v nvidia-smi >/dev/null 2>&1 || ! nvidia-smi -L >/dev/null 2>&1; then
  echo "WARNING: no NVIDIA GPU visible (in Docker: --gpus all, and nvidia-ctk runtime configure --runtime=docker)"
fi

# --- 3. the build products -----------------------------------------------------------------------
[ -f "$TREE/vllm/__init__.py" ] || { echo "TREE=$TREE is not the serving tree"; exit 1; }
[ -f "$TREE/.build-products" ] && [ -s "$TREE/.build-products.sha256" ] \
  || { echo "TREE=$TREE has no verified build products (./build.sh)"; exit 1; }
(cd "$TREE" && sha256sum -c --quiet --strict .build-products.sha256) \
  || { echo "TREE=$TREE: build products changed since ./build.sh verified them"; exit 1; }
[ -x "$VENV/bin/python" ] || { echo "VENV=$VENV has no bin/python"; exit 1; }

# A vllm/ in the launch directory shadows $TREE: `python -m` puts CWD before PYTHONPATH.
if [ -f "$PWD/vllm/__init__.py" ] && [ "$(cd "$PWD/vllm" 2>/dev/null && pwd -P)" != "$(cd "$TREE/vllm" && pwd -P)" ]; then
  echo "refusing to start: $PWD/vllm would shadow $TREE/vllm; start from another directory"; exit 1
fi

# --- 4. first-serve toolchain (FlashInfer JITs against CUDA_HOME, Triton against Python.h) -------
if [ -z "${CUDA_HOME:-}" ]; then
  CUDA_HOME="$("$VENV/bin/python" -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')/nvidia/cu13"
fi
export CUDA_HOME PATH="$CUDA_HOME/bin:$PATH"
for t in cc c++ ninja; do command -v "$t" >/dev/null || { echo "first serve needs $t (apt install gcc g++ ninja-build)"; exit 1; }; done
"$VENV/bin/python" -c 'import os,sys,sysconfig; sys.exit(0 if os.path.exists(os.path.join(sysconfig.get_paths()["include"],"Python.h")) else 1)' \
  || { echo "first serve needs the Python 3.13 headers (apt install python3.13-dev)"; exit 1; }
[ -x "$CUDA_HOME/bin/nvcc" ] || { echo "no nvcc at CUDA_HOME=$CUDA_HOME"; exit 1; }
[ -e "$CUDA_HOME/lib64/libcudart.so" ] || { echo "$CUDA_HOME/lib64/libcudart.so missing (./build.sh creates it)"; exit 1; }
nv_rel="$("$CUDA_HOME/bin/nvcc" --version | sed -n 's/.*release \([0-9][0-9]*\.[0-9][0-9]*\),.*/\1/p' || true)"
[ -n "$nv_rel" ] || { echo "no CUDA release from \`$CUDA_HOME/bin/nvcc --version\`"; exit 1; }
drv_rel="$(nvidia-smi 2>/dev/null | sed -n 's/.*CUDA \(UMD \)\{0,1\}Version: *\([0-9][0-9]*\.[0-9][0-9]*\).*/\2/p' | head -1 || true)"
[ -n "$drv_rel" ] || echo "note: driver CUDA version unreadable; the nvcc-vs-driver check is skipped"
if [ -n "$drv_rel" ] && [ "$(printf '%s\n%s\n' "$nv_rel" "$drv_rel" | sort -V | tail -1)" != "$drv_rel" ]; then
  echo "nvcc CUDA $nv_rel is newer than the driver's CUDA $drv_rel: first-serve kernels would be rejected."; exit 1
fi

# --- 5. topology (opt-in; default = the fixed shipped settings) ----------------------------------
case "${AUTO_TOPO:-0}" in
  0) topo_on=0 ;;
  1 | strict) topo_on=1 ;;
  *) echo "topo: note: AUTO_TOPO='$AUTO_TOPO' is not 0, 1 or strict; the selector stays off"; topo_on=0 ;;
esac
topo_launch_prefix=""
legacy_topo_env() {
  if [ "${1:-}" = fallback ]; then
    if [ "${TOPO_CALLER_SET_CUDA_VISIBLE_DEVICES:-0}" = 1 ]; then CUDA_VISIBLE_DEVICES="$TOPO_CALLER_CUDA_VISIBLE_DEVICES"
    else unset CUDA_VISIBLE_DEVICES; fi
    case "${CUDA_VISIBLE_DEVICES:-}" in *[![:space:]]*) ;; *) unset CUDA_VISIBLE_DEVICES ;; esac
    export TOPO_LAUNCH_PREFIX="" TOPO_DECISION_JSON=""; topo_launch_prefix=""
  fi
  export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3}"
  export CUDA_DEVICE_ORDER=PCI_BUS_ID
  export VLLM_SKIP_P2P_CHECK=1 NCCL_P2P_LEVEL=SYS NCCL_PROTO=LL
}
topo_env_file_ok() {
  local f="$1" i=0 line
  local val="([A-Za-z0-9_@%+=:,./-]+|'[^']*'(\"'\"'[^']*')*)"
  local -a want=(
    "^# topo_select v1 status=(ok|legacy) reason=[A-Za-z0-9_.-]+$"
    "^export CUDA_VISIBLE_DEVICES=$val$"
    "^export CUDA_DEVICE_ORDER=$val$"
    "^export VLLM_SKIP_P2P_CHECK=$val$"
    "^(export NCCL_P2P_LEVEL=$val|unset NCCL_P2P_LEVEL)$"
    "^(export NCCL_PROTO=$val|unset NCCL_PROTO)$"
    "^export TOPO_LAUNCH_PREFIX=('numactl --physcpubind=[0-9,-]+ --preferred=[0-9]+ --'|'')$"
    "^export TOPO_DECISION_JSON=$val$")
  [ -f "$f" ] || return 1
  while IFS= read -r line || [ -n "$line" ]; do
    [ "$i" -lt "${#want[@]}" ] || return 1
    [[ $line =~ ${want[$i]} ]] || return 1
    i=$((i + 1))
  done < "$f" || return 1
  [ "$i" -eq "${#want[@]}" ]
}
topo_unavailable() {
  if [ "$AUTO_TOPO" = strict ]; then echo "topo: $1; AUTO_TOPO=strict refuses to guess"; exit 1; fi
  echo "topo: $1, using legacy settings"; legacy_topo_env fallback
}
if [ "$topo_on" = 0 ]; then
  legacy_topo_env
else
  for v in CUDA_VISIBLE_DEVICES CUDA_DEVICE_ORDER VLLM_SKIP_P2P_CHECK NCCL_P2P_LEVEL NCCL_PROTO PLE_HOME; do
    if [ "${!v+set}" = set ]; then export "TOPO_CALLER_SET_$v=1" "TOPO_CALLER_$v=${!v}"
    else export "TOPO_CALLER_SET_$v=0"; fi
  done
  export TOPO_LAUNCH_PREFIX="" TOPO_DECISION_JSON=""
  topo_timeout=20
  case "${TOPO_TIMEOUT:-}" in
    [1-9]|[1-9][0-9]|[1-9][0-9][0-9]) [ "$TOPO_TIMEOUT" -le 600 ] && topo_timeout="$TOPO_TIMEOUT" ;;
  esac
  topo_json_dir="${CACHE_ROOT:-$HERE/.vllm-cache}/topology"
  tmp_topo_env=""
  if tmp_topo_env="$(mktemp 2>/dev/null)" && [ -n "$tmp_topo_env" ]; then
    trap 'rm -f -- "$tmp_topo_env" 2>/dev/null || :' EXIT
    rc=0
    TOPO_TIMEOUT="$topo_timeout" timeout --kill-after=5 "$((topo_timeout + 5))" \
      "$VENV/bin/python" "$HERE/scripts/topo_select.py" --env-out "$tmp_topo_env" --json-dir "$topo_json_dir" || rc=$?
    if [ "$rc" = 3 ]; then exit 1
    elif [ "$rc" != 0 ]; then topo_unavailable "selector unavailable (rc=$rc)"
    elif ! topo_env_file_ok "$tmp_topo_env"; then topo_unavailable "selector handed back an unusable env file"
    else
      # shellcheck disable=SC1090
      if ! source "$tmp_topo_env"; then topo_unavailable "could not apply the selector's env file"
      else topo_launch_prefix="$TOPO_LAUNCH_PREFIX"; fi
    fi
    rm -f -- "$tmp_topo_env" 2>/dev/null || :; trap - EXIT
  else
    topo_unavailable "could not create a temporary file"
  fi
fi
if [ "${SERVE_PRINT_ENV_AND_EXIT:-0}" = 1 ]; then
  echo "CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-} CUDA_DEVICE_ORDER=${CUDA_DEVICE_ORDER:-} VLLM_SKIP_P2P_CHECK=${VLLM_SKIP_P2P_CHECK:-} NCCL_P2P_LEVEL=${NCCL_P2P_LEVEL-<unset>} NCCL_PROTO=${NCCL_PROTO-<unset>} PREFIX=$topo_launch_prefix"
  exit 0
fi
# --- end topology --------------------------------------------------------------------------------

# --- 6. the serve profile -------------------------------------------------------------------------
export VLLM_PP_LAYER_PARTITION=25,23
# the shipped profile sets every tree knob explicitly; drop inherited ones first
for v in $(env | sed -n 's/^\(MERLIN_[A-Z0-9_]*\|VLLM_E2_[A-Z0-9_]*\)=.*/\1/p'); do unset "$v"; done
unset VLLM_PLE_CPU_OFFLOAD VLLM_PLE_FP8_GLOBAL_SCALE VLLM_PLE_TEST_FAULT VLLM_PLE_TEST_FAULT_GATE VLLM_PLE_TEST_FAULT_LOG \
      VLLM_QSA_KV_COLLECT VLLM_QSA_KV_CLIP_COUNT VLLM_E2_ALLOW_BREAKABLE VLLM_E2_EXPECTED_CUDAGRAPH_MODE \
      VLLM_E2_MAMBA_SLOT_OWNERSHIP VLLM_E2_MEMSTATS VLLM_E2_MEMSTATS_DIR VLLM_E2_PLE_FLAG_CHECK \
      VLLM_E2_SCHED_LOG VLLM_E2_ZERO_OWNERSHIP_CHECK VLLM_E2_GUARD_INTERVAL \
      VLLM_PLE_OFFLOAD_WAIT_TIMEOUT VLLM_PLE_PROGRESS_DEADLINE_S VLLM_PLE_STARTUP_DEADLINE_S \
      PYTORCH_CUDA_ALLOC_CONF || true
export VLLM_E2_PLE_PULL_TRANSPORT=1 VLLM_PLE_OFFLOAD_READY_TIMEOUT=1800
export VLLM_PLE_OFFLOAD_HOME_DEVICE="${PLE_HOME:-0}"
export VLLM_HOST_EMBED_TABLE=1
export VLLM_USE_BREAKABLE_CUDAGRAPH=0
export VLLM_QSA_KV_SCALES="${SCALES:-$CKPT/qsa_kv_scales.json}"
# Engine stats line (throughput, KV usage, queue). 1 s matches how often the numbers are useful.
# Unset, vLLM waits 10 s (vllm/envs.py:47), so a quiet run looks idle for ten seconds.
export VLLM_LOG_STATS_INTERVAL="${VLLM_LOG_STATS_INTERVAL:-1}"
export HF_HUB_OFFLINE=1 HF_DATASETS_OFFLINE=1
# the kernel toggles (the shipped values). These names are the tree's own: envs.py in the
# serving tree registers MERLIN_* for the kernel toggles and VLLM_E2_* for the runtime knobs.
export MERLIN_FULL_K=1 MERLIN_QSA_SORT=1 MERLIN_TIE_DET=1 MERLIN_TIE_RECENT=1 MERLIN_TOPK_SORTED_EMIT=1
export VLLM_E2_GUARD_MODE="${GUARD:-warn}"
export VLLM_E2_SIDE_CACHE_LAYOUT=1 VLLM_USE_V2_MODEL_RUNNER=1 VLLM_E2_MAMBA_RETAIN_CHECKPOINT=1
# Accept any name in the model field of a request, like llama.cpp. Upstream's own switch, on by
# default here: /v1/models and the model field of every reply still report the served name.
# Set VLLM_SKIP_MODEL_NAME_VALIDATION=0 to reject a name that is not served.
export VLLM_SKIP_MODEL_NAME_VALIDATION="${VLLM_SKIP_MODEL_NAME_VALIDATION:-1}"
if [ "${COUNTERS:-0}" = 1 ]; then export MERLIN_TOPK_EXACT_COUNTERS_LOG=30 VLLM_QSA_KV_CLIP_COUNT=300; fi
[ -f "$VLLM_QSA_KV_SCALES" ] || { echo "scales sidecar missing: $VLLM_QSA_KV_SCALES (FP8 KV would run at scale 1.0). Set SCALES=<path> to the sidecar of this model."; exit 1; }

# Concurrency and the prefill chunk, tunable from .env. vLLM refuses a chunk smaller than the
# sequence count (vllm/config/scheduler.py:309); say it here, where the reader set the value.
MAX_SEQS="${MAX_SEQS:-4}"
MAX_BATCHED_TOKENS="${MAX_BATCHED_TOKENS:-2048}"
for v in MAX_SEQS MAX_BATCHED_TOKENS; do
  case "${!v}" in ''|*[!0-9]*) echo "$v must be a positive integer (got: ${!v})"; exit 1 ;; esac
  [ "${!v}" -ge 1 ] || { echo "$v must be 1 or more (got: ${!v})"; exit 1; }
done
if [ "$MAX_BATCHED_TOKENS" -lt "$MAX_SEQS" ]; then
  echo "MAX_BATCHED_TOKENS ($MAX_BATCHED_TOKENS) must be >= MAX_SEQS ($MAX_SEQS)"; exit 1
fi
# The shipped pool holds 924,993 tokens, pinned by --kv-cache-memory. A shape that needs more does
# not fail: the engine preempts and recomputes.
if [ $((MAX_SEQS * 262144)) -gt 924993 ]; then
  echo "note: MAX_SEQS=$MAX_SEQS wants $((MAX_SEQS * 262144)) tokens at the full 262144 context; the pool holds 924993, so full-length requests preempt"
fi
# CUDA graphs must cover the widest decode batch: MAX_SEQS x (K+1) tokens, K the MTP depth. A
# batch pads up to the next captured size, and runs outside the graph above the largest one. The
# sizes below the MTP width stay for small and mixed batches. Each size costs capture time and
# VRAM, so the list stops at the widest batch, as the shipped shape does (4 x 4 = 16).
# MTP depth: 3 is Qwen 3.8's recommended value, not a knob. The graph math below reads it, so keep
# it equal to the num_speculative_tokens in --speculative-config.
MTP_K=3
CG_TOP=$((MAX_SEQS * (MTP_K + 1)))
CG_SIZES="1,2,4"
for ((n = MTP_K + 1; n <= CG_TOP; n += MTP_K + 1)); do
  [ "$n" -gt 4 ] && CG_SIZES="$CG_SIZES,$n"
done
CG_COUNT=$((2 + MAX_SEQS))
if [ "$CG_COUNT" -gt 8 ]; then
  echo "note: $CG_COUNT CUDA graphs for MAX_SEQS=$MAX_SEQS; capture time and graph VRAM both grow with the count"
fi
echo "cuda graphs: 1,2,4 then every $((MTP_K + 1)) tokens to $CG_TOP"

# Compile cache: on disk by default, so it survives a reboot. A cold cache costs ~6 min of kernel
# compile, and the cache is only ~160 MB, so disk is the right home for it. Keep ONE root per
# deployment -- a new root costs a fresh compile. CACHE_ROOT=/vllm-cache (a tmpfs) is the opt-in
# for a box where you want it in RAM.
[ -d "$CACHE_ROOT" ] || { mkdir -p "$CACHE_ROOT" 2>/dev/null || { echo "cannot create $CACHE_ROOT"; exit 1; }; }
if df -T "$CACHE_ROOT" 2>/dev/null | grep -q tmpfs; then
  echo "note: $CACHE_ROOT is a tmpfs; the compile cache is lost on reboot"
fi
export VLLM_CACHE_ROOT="$CACHE_ROOT"
export FLASHINFER_WORKSPACE_BASE="$CACHE_ROOT/flashinfer" TRITON_CACHE_DIR="$CACHE_ROOT/triton"
export PYTHONPATH="$TREE" VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0

# host RAM (the n-gram table lives here; a short host OOMs mid-boot) and /dev/shm
mem_kb="$(awk '/^MemTotal:/{print $2}' /proc/meminfo 2>/dev/null || true)"
for f in /sys/fs/cgroup/memory.max /sys/fs/cgroup/memory/memory.limit_in_bytes; do
  lim="$(cat "$f" 2>/dev/null || true)"
  case "$lim" in ''|*[!0-9]*) continue ;; esac
  lim_kb=$((lim / 1024)); if [ -z "${mem_kb:-}" ] || [ "$lim_kb" -lt "$mem_kb" ]; then mem_kb="$lim_kb"; fi
done
if [ -n "${mem_kb:-}" ] && [ "$mem_kb" -lt 100663296 ]; then
  echo "WARNING: ${mem_kb} KB visible RAM (< the required 96 GiB); resident floor is ~69 GiB"
fi
shm_kb="$(df -Pk /dev/shm 2>/dev/null | awk 'NR==2{print $2}' || true)"
case "$shm_kb" in *[!0-9]*) shm_kb= ;; esac
if [ -n "${shm_kb:-}" ] && [ "$shm_kb" -lt 1048576 ]; then
  echo "WARNING: /dev/shm is $((shm_kb / 1024)) MB; the PLE transport needs >= 1 GB"
fi

read -r -a NAMES <<< "${MODEL_NAME:-qwen38-flash-next}"
echo "serving $CKPT as ${NAMES[*]} on ${HOST:-127.0.0.1}:${PORT:-8000} (cache $CACHE_ROOT)"
if [ -n "$LOG" ]; then
  # stdout + stderr, so the PLE offload child and the engine workers land in the same file.
  # systemd/vllm.logrotate rotates it: daily, 100 MB cap, 14 files, gzip. Read it with: tail -f $LOG
  touch "$LOG" 2>/dev/null || { echo "cannot write $LOG; run as root or set LOG=<path>"; exit 1; }
  echo "log: $LOG (rotate: systemd/vllm.logrotate)"
  exec >> "$LOG" 2>&1
fi

# unquoted on purpose: word-splits into a numactl prefix from the validated selector file, empty when off
exec ${topo_launch_prefix:+$topo_launch_prefix} "$VENV/bin/vllm" serve "$CKPT" \
  --served-model-name "${NAMES[@]}" \
  --host "${HOST:-127.0.0.1}" --port "${PORT:-8000}" \
  --tensor-parallel-size 2 --pipeline-parallel-size 2 --enable-expert-parallel \
  --speculative-config '{"method":"mtp","num_speculative_tokens":3,"draft_sample_method":"probabilistic"}' \
  --max-model-len 262144 --max-num-seqs "$MAX_SEQS" --max-num-batched-tokens "$MAX_BATCHED_TOKENS" \
  --gpu-memory-utilization 0.96 --kv-cache-memory 4100000000 --kv-cache-dtype fp8_e4m3 \
  --offload-towers --limit-mm-per-prompt '{"image":42,"video":0}' \
  --mm-processor-kwargs '{"size":{"longest_edge":4194304,"shortest_edge":65536}}' \
  --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":['"$CG_SIZES"']}' \
  --enable-prefix-caching --enable-prompt-tokens-details \
  --no-enable-flashinfer-autotune \
  --enable-auto-tool-choice --tool-call-parser qwen3_coder --reasoning-parser qwen3 \
  --override-generation-config '{"temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0}' \
  --default-chat-template-kwargs '{"enable_thinking": true, "reasoning_effort": "xhigh"}' \
  --shutdown-timeout 10 \
  --block-size 2048 --use-replayssm \
  --prefix-match-unit 64 \
  --safetensors-load-strategy prefetch \
  --disable-uvicorn-access-log \
  "$@"