#!/usr/bin/env bash

# Build the serving tree: vLLM 0.30.0 + patches/, compiled for 4x RTX 3090.
#
#   ./build.sh [--force]
#   ARTIFACTS=artifacts/build-artifacts-<tag>.tar.gz ./build.sh
#
# Default: everything is compiled here, from this tree's source. No flags, no route to choose.
#   step 2  creates ./vllm (gitignored): upstream vLLM v0.30.0 (ced6857afa) + patches/*.patch
#           (100 patches), then asserts the result against the expected tree hash. FETCH=0 reuses the
#           existing tree; FETCH=1 re-applies the series from the base.
#   step 4  a fresh .venv from CPython 3.13 + scripts/requirements-pinned.txt (torch 2.13.0, CUDA 13.0).
#   step 6  the CUDA 13.0 compiler from the venv's nvidia wheels, targets arch 8.6 (sm_86 = RTX 3090).
#   step 7  uv pip install -e . --no-deps --no-build-isolation: configures with cmake, compiles every
#           extension of this tree (CUDA/C++/PTX, plus the Rust frontend when a Rust toolchain exists),
#           installs the tree into .venv in place, and gives the tree its version and console script.
#
# ARTIFACTS=<tarball> installs compiled kernels of this same tree instead of compiling them. step 6
# unpacks them after it checks the compiled inputs (OPS_OBJ below), the arch, the CPython minor,
# the CUDA major and the torch version in the tarball's manifest, and hashes every file after the
# unpack. step 7 then installs the tree without compiling (VLLM_TARGET_DEVICE=empty). step 8 writes
# that tarball, so the Dockerfile uses this route when you pass --build-context ops=$PWD/artifacts.
#
# step 8 packs the kernels into artifacts/ for the image build (PACK=0 or --no-pack skips it; the
# Dockerfile sets it, an image build must not grow a 340 MB tarball inside a layer). Delete artifacts/
# and the next run writes it again from the compiled products, with no recompile.
#
# Needs: git curl tar gzip sha256sum cmake >= 3.26 ninja cc c++, and uv (UV=<path>, UV=0 to refuse)
# or python3.13 with the venv module. Optional: cargo 1.95 for the Rust frontend.
# Budget: ~30 GB free disk, ~20 min on 64 cores for a first build. MAX_JOBS caps the compile jobs
# (default: every core; each nvcc job can use 4 GB RAM).
#
# Re-runs: a finished build is recorded in vllm/.build-products (tree head + the two extension
# hashes) and vllm/.build-products.sha256 (every build product). When the tree has not moved, every
# hash still matches, and .venv holds the install, steps 6 and 7 are skipped. A tree move, a changed
# product, a missing product, or a missing install ends the skip. --force (or FORCE=1) skips nothing
# and compiles again.
#
# ccache: used automatically when it is on PATH. vllm/setup.py passes ccache as the cmake compiler
# launcher for C, C++, CUDA and HIP, so a rebuild after a patch change recompiles only the files
# that changed. Objects go to the ccache directory of the current user ($HOME/.ccache by default,
# CCACHE_DIR=<path> to move it). A first build fills the cache and gains nothing; the second one is
# where it pays. Without ccache every kernel compiles from scratch and build.sh says so.

set -euo pipefail

STEPS=8
FORCE="${FORCE:-0}"
PACK="${PACK:-1}"
for opt in "$@"; do
  case "$opt" in
    --force|-f) FORCE=1 ;;
    --no-pack) PACK=0 ;;
    -h|--help) echo "usage: ./build.sh [--force] [--no-pack]"; echo "       --force    compile again even when nothing changed"; echo "       --no-pack  do not write the compiled kernels to artifacts/"; echo "       ARTIFACTS=<tarball>  install compiled kernels from artifacts/ instead of compiling"; exit 0 ;;
    *) echo "unknown option: $opt  (./build.sh --help)"; exit 1 ;;
  esac
done

# Every step announces itself before it runs, so a stall or a failure names the step.
step(){ printf '\n==> [%s/%s] %s\n' "$1" "$STEPS" "$2"; }

step 1 "tools and CPython 3.13"
for t in git curl tar gzip sha256sum cmake ninja cc c++; do command -v "$t" >/dev/null || { echo "missing tool: $t"; exit 1; }; done
cm="$(cmake --version | sed -n 's/^cmake version \([0-9][0-9.]*\).*/\1/p' || true)"
[ -n "$cm" ] || { echo "no cmake version from \`cmake --version\`"; exit 1; }
[ "$(printf '%s\n3.26\n' "$cm" | sort -V | head -1)" = 3.26 ] || { echo "building from source needs cmake >= 3.26 (found $cm)"; exit 1; }
PY=python3.13
UV="${UV:-}"; [ -n "$UV" ] || UV="$(command -v uv || true)"
if [ -n "$UV" ]; then
  "$UV" python find 3.13 >/dev/null 2>&1 || "$UV" python install 3.13 || { echo "$UV cannot provide CPython 3.13"; exit 1; }
  PY="$("$UV" python find 3.13)"
elif command -v python3.13 >/dev/null; then
  probe="$(mktemp -d)"; trap 'rm -rf "$probe"' EXIT
  python3.13 -m venv "$probe/p" >/dev/null 2>&1 || { echo "python3.13 cannot create a venv: apt install python3.13-venv"; exit 1; }
  rm -rf "$probe"; trap - EXIT
else
  echo "need uv (https://docs.astral.sh/uv) or python3.13 with the venv module"; exit 1
fi
HERE="$(cd "$(dirname "$0")" && pwd)"; ROOT="$HERE"
abspath(){ case "$1" in /*) printf '%s\n' "$1" ;; *) printf '%s\n' "$PWD/$1" ;; esac; }
SRC="$(abspath "${SRC:-$HERE/vllm}")"
VENV="$(abspath "${VENV:-$HERE/.venv}")"
ARCH="${ARCH:-8.6}"
JOBS="${MAX_JOBS:-$(nproc)}"
cc=""
if command -v ccache >/dev/null; then
  cc="$(command -v ccache)"
  # CCACHE_DIR is left alone: ccache uses its own per-user directory ($HOME/.cache/ccache,
  # /root/.cache/ccache for root), so the cache is shared with every other build on this account.
  [ -n "${CCACHE_DIR:-}" ] && mkdir -p "$CCACHE_DIR"
  ccd="$( "$cc" --show-config 2>/dev/null | sed -n 's/.*cache_dir = //p' | head -1 )"
  [ -n "$ccd" ] || ccd="${CCACHE_DIR:-the ccache default for this user}"
fi
echo "    repo=$ROOT SRC=$SRC VENV=$VENV"
echo "    python: $PY  cmake $cm  arch=$ARCH  MAX_JOBS=$JOBS"
if [ -n "$cc" ]; then
  echo "    ccache: $cc  dir=$ccd  (objects reused on the next build)"
else
  echo "    ccache: not on PATH, every kernel compiles from scratch"
fi

step 2 "serving tree (upstream v0.30.0 + $(ls "$HERE"/patches/*.patch | wc -l) patches)"
# The tree is not in this repo: this repo ships the patches, not the tree. Fetch applies the series
# to the upstream tag and asserts the result, so a patch series that drifts fails here, not at serve.
BASE=ced6857afa0ea7b2e3f0846a62e1394e90f15607              # tag v0.30.0
EXPECTED_TREE=a823fa36d3fd640f87b3de87153268d938130a47     # BASE + patches/*.patch (106), verified 2026-10-18
# The tree carries no build record: the series of patch 0103 once committed .build-products.sha256,
# which build.sh rewrites, so every re-run failed the "modified tracked files" check below.
URL="${VLLM_UPSTREAM_URL:-https://github.com/vllm-project/vllm.git}"
# git am makes a commit, so git needs a committer identity: the host's ~/.gitconfig is not there (a
# container, a build user, a fresh box). The committer is not in the tree hash, so the assertion
# below does not depend on it.
GITID=( -c user.name="Hinata" -c user.email="hinata@idihore.id" )
git config --global core.autocrlf false 2>/dev/null || true
if [ "${FETCH:-0}" = 1 ] || [ ! -f "$SRC/vllm/__init__.py" ]; then
  if [ -e "$SRC" ] && [ ! -d "$SRC/.git" ] && [ ! -f "$SRC/vllm/__init__.py" ]; then
    echo "$SRC exists and is not a git checkout; move it away first"; exit 1
  fi
  echo "    base=$BASE tag v0.30.0"
  echo "    patches=$(ls "$HERE"/patches/*.patch | wc -l) from $HERE/patches"
  echo "    upstream=$URL"
  [ -d "$SRC/.git" ] || { echo "    git init $SRC"; git init -q "$SRC"; }
  git -C "$SRC" remote add upstream "$URL" 2>/dev/null || git -C "$SRC" remote set-url upstream "$URL"
  if ! git -C "$SRC" cat-file -e "$BASE^{commit}" 2>/dev/null; then
    echo "    fetching v0.30.0 (shallow, ~200 MB, this is the slow part on a fresh host)..."
    git -C "$SRC" fetch --quiet --depth 1 "$URL" "refs/tags/v0.30.0:refs/heads/flashnext" \
      || { echo "fetch v0.30.0 from $URL failed (no network?)"; exit 1; }
  fi
  git -C "$SRC" checkout -q flashnext 2>/dev/null || git -C "$SRC" checkout -q -b flashnext
  git -C "$SRC" reset -q --hard "$BASE"
  echo "    applying patches (git am)..."
  git -C "$SRC" "${GITID[@]}" am --keep-non-patch "$HERE"/patches/*.patch >/dev/null
fi
tree="$(git -C "$SRC" rev-parse 'HEAD^{tree}' 2>/dev/null || echo none)"
[ "$tree" = "$EXPECTED_TREE" ] || { echo "tree $tree != expected $EXPECTED_TREE: the series drifted, or the tree is older than the patches. FETCH=1 re-applies it"; exit 1; }
echo "    tree ready: $(git -C "$SRC" rev-parse --short HEAD) tree $tree"
# build.sh writes the build record (MARK/SUMS below) inside the tree, so those two paths are never a
# change to the series. Excluding them keeps this check for real drift only, and names the file.
dirty="$(git -C "$SRC" status --porcelain --untracked-files=no -- . ':(exclude).build-products' ':(exclude).build-products.sha256' 2>/dev/null)"
[ -n "$dirty" ] && { echo "$SRC has modified tracked files:"; printf '%s\n' "$dirty" | sed 's/^/      /'; exit 1; }
TREE_HEAD="$(git -C "$SRC" rev-parse --short HEAD 2>/dev/null || echo unknown)"
# The commit SHA of the same tree differs between machines: git am stamps the committer date of the
# box that applied the patches, so the host commit and the container commit of one patch set are
# not the same SHA. The tree object is identical. Compare that, not the SHA.
TREE_OBJ="$(git -C "$SRC" rev-parse 'HEAD^{tree}' 2>/dev/null || echo unknown)"
# The ops cache key: the compiled inputs only. A patch that touches Python alone leaves this key
# alone, so a tarball of prebuilt kernels stays valid for it and the image build keeps the ~2 min
# route. A patch that moves any path below invalidates the tarball, and the build compiles.
# A path missing from the tree yields "missing", which no manifest can match: the fail-safe way.
OPS_PATHS="csrc CMakeLists.txt cmake setup.py pyproject.toml MANIFEST.in requirements tools rust-toolchain.toml vllm/third_party vllm/vllm_flash_attn"
OPS_OBJ="$(for p in $OPS_PATHS; do git -C "$SRC" rev-parse -q --verify "HEAD^{tree}:$p" 2>/dev/null || echo missing; done | LC_ALL=C sha256sum | cut -d' ' -f1)"
MARK="$SRC/.build-products"; SUMS="$SRC/.build-products.sha256"
LIST="$HERE/scripts/build-artifacts.list"

step 3 "venv (fresh, never reused)"
if [ -e "$VENV" ] && [ ! -f "$VENV/.build-created" ]; then echo "$VENV exists and was not created here; pick a fresh VENV path"; exit 1; fi
if [ ! -x "$VENV/bin/python" ]; then
  if [ -n "$UV" ]; then
    v="$("$UV" venv --python 3.13 "$VENV" 2>&1)" || { echo "uv venv failed: $v"; exit 1; }
    echo "    $v"
  else
    "$PY" -m venv "$VENV"
  fi
  echo "created by build.sh" > "$VENV/.build-created"
else
  echo "    venv exists, reusing: $VENV"
fi
if [ -x "$VENV/bin/pip" ]; then
  "$VENV/bin/python" -m pip install pip==26.2.1
  PIP_INSTALL="$VENV/bin/pip install"
elif [ -n "$UV" ]; then
  PIP_INSTALL="$UV pip install --python $VENV/bin/python"
else
  echo "$VENV has no pip and no uv is available"; exit 1
fi
echo "    installer: $PIP_INSTALL"
# Where the wheels come from. uv and pip use the cache directory of the current user, so a build
# under the same account reuses the wheels of every previous build. Nothing here changes them.
cache_of(){ [ -n "${2:-}" ] && { printf '%s\n' "$2"; return; }; printf '%s\n' "${HOME:-/root}/.cache/$1"; }
uv_cd="${UV_CACHE_DIR:-$(cache_of uv "${XDG_CACHE_HOME:-}")}"
if [ -n "$UV" ]; then
  uv_n="$(du -sh "$uv_cd" 2>/dev/null | cut -f1 || true)"
  echo "    uv cache: $uv_cd${uv_n:+ ($uv_n)}"
fi
pip_cd="${PIP_CACHE_DIR:-$(cache_of pip "${XDG_CACHE_HOME:-}")}"
pip_n="$(du -sh "$pip_cd" 2>/dev/null | cut -f1 || true)"
echo "    pip cache: $pip_cd${pip_n:+ ($pip_n)}"
pip_install(){ $PIP_INSTALL "$@"; }

step 4 "pinned runtime (torch 2.13.0 + CUDA 13.0 and the vllm deps; $(grep -cv '^\s*\(#\|$\)' "$HERE/scripts/requirements-pinned.txt") pinned packages)"
pip_install -r "${REQ:-$HERE/scripts/requirements-pinned.txt}"
pip_install setuptools-rust==1.13.0 setuptools-scm==10.3.4 vcs-versioning==2.5.0 semantic-version==2.10.0
"$VENV/bin/python" -c "import torch; assert torch.__version__.split('+')[0] == '2.13.0' and torch.version.cuda == '13.0', (torch.__version__, torch.version.cuda)"

step 5 "compile toolchain (CUDA 13.0 from the venv wheels)"
CU="$("$VENV/bin/python" -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')/nvidia/cu13"
[ -x "$CU/bin/nvcc" ] || { echo "no CUDA toolkit wheel at $CU"; exit 1; }
[ -e "$CU/lib64" ] || ln -s lib "$CU/lib64"
for so in "$CU"/lib/lib*.so.[0-9]*; do base="${so%%.so.*}.so"; [ -e "$base" ] || ln -s "$(basename "$so")" "$base"; done
[ -e "$CU/lib/libcudart.so" ] || { echo "$CU/lib has no libcudart.so.13"; exit 1; }
NVCC="${NVCC:-$CU/bin/nvcc}"
nv_rel="$("$NVCC" --version | sed -n 's/.*release \([0-9][0-9]*\.[0-9][0-9]*\),.*/\1/p')"
[ "$nv_rel" = 13.0 ] || { echo "nvcc is CUDA ${nv_rel:-unknown}, expected 13.0"; exit 1; }
echo "    nvcc $nv_rel at $NVCC (start.sh points CUDA_HOME here, for the runtime JIT kernels)"

# The kernels link CUDA::cuda_driver, the driver library (libcuda.so.1). The
# driver package provides it; the CUDA wheel does not. Generate the link stub.
stub_dir="$CU/lib/stubs"
mkdir -p "$stub_dir"
"$HERE/scripts/gen-cuda-stub.sh" "$CU/include/cuda.h" "$stub_dir/libcuda.so" | sed 's/^/    /'
cargo="$(command -v cargo || true)"
if [ -n "$cargo" ]; then
  echo "    rust frontend: $cargo $(cargo --version 2>/dev/null || echo unknown) -- vllm-rs and _rust_tool_parser will be built"
else
  echo "    rust frontend: no cargo -> vllm-rs and _rust_tool_parser are skipped."
  echo "      the shipped serve profile uses the Python tool parsers, so serving is unaffected."
  echo "      install rustup toolchain 1.95 (vllm/rust-toolchain.toml) and re-run ./build.sh to add them."
fi

# glibc: the products of a packed build carry the newest glibc symbol they need. The box that
# installs them must be at least that new, or the loader fails at serve time, not here. Print it
# and stop, with the reason.
glibc_ver(){ ldd --version 2>/dev/null | sed -n 's/^ldd [^0-9]*\([0-9][0-9.]*\).*/\1/p' | head -1; }
ver_ge(){ [ "$(printf '%s\n%s\n' "$1" "$2" | sort -V | head -1)" = "$2" ]; }

# The build products this tree must have at serve time. build-artifacts.list is the list.
check_list(){ local miss=0 p; while read -r p; do case "$p" in ''|'#'*) continue ;; esac; [ -e "$SRC/${p%/}" ] || { echo "missing build product: $p"; miss=1; }; done < "$LIST"; return $miss; }
write_sums(){ (cd "$SRC" && while read -r p; do case "$p" in ''|'#'*) continue ;; esac; find "${p%/}" -type f; done < "$LIST" | grep -vx 'vllm/_version.py' | LC_ALL=C sort | xargs -d '\n' sha256sum) > "$SUMS.tmp" && mv "$SUMS.tmp" "$SUMS"; }
check_sums(){ [ -s "$SUMS" ] && (cd "$SRC" && sha256sum -c --quiet --strict "$SUMS"); }

# Prebuilt ops: ARTIFACTS=<tarball> installs compiled kernels instead of compiling them. The
# tarball comes from step 8 of a finished build of this exact tree. The checks below are the reason
# a packed build is safe to install: the tree
# head, the CPython minor, the CUDA major, the torch version and the arch must all match, and
# every file is hashed after the unpack. One mismatch stops the build and says which one.
prebuilt=0
if [ -n "${ARTIFACTS:-}" ]; then
  step 6 "install prebuilt ops (no compile)"
  [ -f "$ARTIFACTS" ] || { echo "ARTIFACTS=$ARTIFACTS is not a file"; exit 1; }
  adir="$(cd "$(dirname "$ARTIFACTS")" && pwd)"; abase="$(basename "$ARTIFACTS")"; stem="${abase%.tar.gz}"
  [ -f "$adir/$stem.sha256" ] || { echo "$adir/$stem.sha256 is needed: the per-file hashes verified after the unpack"; exit 1; }
  [ -f "$adir/$stem.manifest" ] || { echo "$adir/$stem.manifest is needed: it records the tree, arch, python, CUDA and torch the ops were built for"; exit 1; }
  if [ -f "$ARTIFACTS.sha256" ]; then
    ( cd "$adir" && sha256sum -c --quiet "$stem.tar.gz.sha256" ) || { echo "$ARTIFACTS does not match $stem.tar.gz.sha256"; exit 1; }
    echo "    tarball sha256: ok"
  else
    echo "    tarball sha256: no $stem.tar.gz.sha256 sidecar, the tarball itself is not verified"
  fi
  # Read the manifest with sed, not with ".": cc= and glibc= carry text with parentheses and
  # spaces, which the shell cannot source.
  mf(){ sed -n "s/^$1=//p" "$adir/$stem.manifest" | head -1; }
  m_tree="$(mf tree)"; m_tree_obj="$(mf tree_obj)"; m_arch="$(mf arch)"; m_py="$(mf python)"; m_cuda="$(mf cuda)"
  m_torch="$(mf torch)"; m_glibc="$(mf glibc)"; m_files="$(mf files)"; m_built="$(mf built)"; m_ops_obj="$(mf ops_obj)"
  echo "    manifest: tree=$m_tree tree_obj=${m_tree_obj:0:7} ops_obj=${m_ops_obj:0:7} arch=$m_arch python=$m_py cuda=$m_cuda torch=$m_torch glibc=$m_glibc files=$m_files built=$m_built"
  echo "    this tree: tree_obj=${TREE_OBJ:0:7} ops_obj=${OPS_OBJ:0:7}"
  fail=0
  [ -n "$m_tree_obj" ] || { echo "$adir/$stem.manifest has no tree_obj= line: pack the ops again with ./build.sh"; exit 1; }
  # The compiled inputs are what the kernels were built from, so OPS_OBJ is the check. A Python-only
  # patch moves the whole tree object, and must not push this build into a 20-40 min compile. The
  # commit SHA is printed next to it and is allowed to differ between machines (committer date).
  if [ -z "$m_ops_obj" ]; then
    echo "    manifest has no ops_obj= line, it predates this check: the strict tree_obj test applies"
    m_ops_obj="$m_tree_obj"
  fi
  [ "$m_ops_obj" = "$OPS_OBJ" ] || { echo "the ops were compiled from inputs ${m_ops_obj:0:7}, this tree compiles from ${OPS_OBJ:0:7} (trees $m_tree_obj / $TREE_OBJ, commits $m_tree / $TREE_HEAD): pack them from this tree, or compile"; fail=1; }
  venv_py="$("$VENV/bin/python" -c 'import sys; print("%d.%d" % sys.version_info[:2])')"
  [ "$m_py" = "$venv_py" ] || { echo "the ops are for CPython $m_py, the venv is $venv_py"; fail=1; }
  [ "$m_cuda" = "$nv_rel" ] || { echo "the ops are for CUDA $m_cuda, this build uses CUDA $nv_rel"; fail=1; }
  venv_torch="$("$VENV/bin/python" -c 'import torch; print(torch.__version__.split("+")[0])')"
  [ "$m_torch" = "$venv_torch" ] || { echo "the ops are for torch $m_torch, the venv has torch $venv_torch"; fail=1; }
  [ "$m_arch" = "$ARCH" ] || { echo "the ops are built for arch $m_arch, this build asks for arch $ARCH"; fail=1; }
  [ "$fail" = 0 ] || exit 1
  if [ -n "${FORCE:-}" ] && [ "$FORCE" = 1 ]; then rm -f "$MARK" "$SUMS"; echo "    --force: replacing the products in the tree"; fi
  echo "    unpacking $(du -h "$ARTIFACTS" | cut -f1) into $SRC"
  tar -xzf "$ARTIFACTS" -C "$SRC"
  ( cd "$SRC" && sha256sum -c --quiet --strict "$adir/$stem.sha256" ) \
    || { echo "a file of the tarball does not match $stem.sha256 after the unpack"; exit 1; }
  echo "    unpacked: all $m_files file hashes match the manifest"
  need="$(readelf -V "$SRC/vllm/_C_stable_libtorch.abi3.so" 2>/dev/null | grep -o 'GLIBC_[0-9.]*' | sed 's/GLIBC_//' | sort -V | tail -1)"
  have="$(glibc_ver)"
  if [ -n "$need" ] && [ -n "$have" ]; then
    ver_ge "$have" "$need" || { echo "these ops need glibc $need, this box has $have: build the image on a base with that glibc, or compile here"; exit 1; }
    echo "    glibc: ops need $need, this box has $have"
  else
    echo "    glibc check skipped: no readelf or no ldd here"
  fi
  check_list || { echo "the tarball does not carry every product of $(basename "$LIST")"; exit 1; }
  write_sums
  prebuilt=1
fi

# A finished build of this tree is reused, so a re-run does not take 20 minutes again. Every check
# below has to pass to skip the compile, and the one that fails says what changed.
reuse=0
if [ "$prebuilt" = 1 ]; then
  reuse=0
elif [ -f "$MARK" ] && [ "$FORCE" = 1 ]; then
  echo "    --force: compiling again; the record in $(basename "$MARK") is not trusted"
elif [ -f "$MARK" ]; then
  read -r _ c_sha moe_sha tree_at_build < "$MARK"
  reuse=1
  if [ "$tree_at_build" != "$TREE_HEAD" ]; then
    # A tree move only forces an extension rebuild when a C++/CUDA source actually changed.
    # A Python-only patch (the common case) reuses the existing extensions: ninja would
    # otherwise recompile everything because git am refreshes every file timestamp.
    if git -C "$SRC" diff --name-only "$tree_at_build" "$TREE_HEAD" -- 'csrc/' '*.cu' '*.cuh' '*.cpp' '*.cc' 'CMakeLists.txt' 'setup.py' 'pyproject.toml' 2>/dev/null | grep -q .; then
      echo "note: the tree moved $tree_at_build -> $TREE_HEAD and C++/CUDA sources changed; rebuilding the extensions"
      reuse=0
    else
      echo "note: the tree moved $tree_at_build -> $TREE_HEAD but no C++/CUDA sources changed; reusing the extensions"
    fi
  fi
  if [ "$(sha256sum "$SRC/vllm/_C_stable_libtorch.abi3.so" 2>/dev/null | cut -d' ' -f1)" != "$c_sha" ] || [ "$(sha256sum "$SRC/vllm/_moe_C_stable_libtorch.abi3.so" 2>/dev/null | cut -d' ' -f1)" != "$moe_sha" ]; then
    echo "note: the extensions are not the ones recorded in $MARK; the compile below rebuilds them"
    reuse=0
  fi
  check_list || { echo "note: build products are missing; the compile below builds them"; reuse=0; }
  [ "$reuse" = 1 ] && { check_sums || { echo "note: build products changed since $(basename "$SUMS"); the compile below rebuilds them"; reuse=0; }; }
  # A source-only change (a patch that touches no kernel) is invisible to the product hashes above,
  # and the editable install of step 7 is what makes the tree importable. So the install must exist.
  [ "$reuse" = 1 ] && { ls -d "$VENV"/lib/python3.*/site-packages/vllm-*.dist-info >/dev/null 2>&1 \
    || { echo "note: $VENV has no installed vllm; step 7 installs it"; reuse=0; }; }
  if [ "$reuse" = 1 ]; then
    echo "    existing build products: present, all $(wc -l < "$SUMS") hashes re-verified (built at tree $tree_at_build)"
  else
    rm -f "$MARK" "$SUMS"; echo "    old build record removed, verification runs after the compile"
  fi
fi

if [ "$prebuilt" = 1 ]; then
  step 7 "install the tree into .venv (editable, prebuilt ops, metadata only)"
  # The kernels are already compiled, so setup.py must not compile again. VLLM_TARGET_DEVICE=empty
  # makes it skip every compile and write the version file and the console script only.
  build_log="$ROOT/build-$(date +%Y%m%d-%H%M%S).log"
  echo "    full build output -> $build_log"
  t0=$(date +%s)
  ( while sleep 60; do echo "    ... $(( ( $(date +%s) - t0 ) / 60 )) min elapsed, installing metadata"; done ) & hb=$!
  ( cd "$SRC" && env VLLM_TARGET_DEVICE=empty VLLM_VERSION_OVERRIDE=0.30.0 \
      sh -c "$PIP_INSTALL -e . --no-deps --no-build-isolation" ) 2>&1 | tee "$build_log" \
    || { kill "$hb" 2>/dev/null; echo "step 7 failed; the install output is in $build_log"; exit 1; }
  kill "$hb" 2>/dev/null || true; wait "$hb" 2>/dev/null || true
  check_list || { echo "installed, but build products are missing; see $build_log"; exit 1; }
elif [ "$reuse" = 1 ]; then
  step 6 "compile (skipped: no change to the tree, the products, or the install)"
  step 7 "install (skipped: .venv already holds this tree)"
  echo "    nothing to compile. ./build.sh --force compiles again."
else
  step 6 "build this tree from source (arch $ARCH, $(grep -vc '^\s*#\|^\s*$' "$LIST") products in $(basename "$LIST"))"
  echo "    setup.py configures cmake and compiles: $(grep -v '^\s*#\|^\s*$' "$LIST" | tr '\n' ' ')"
  echo "    first build: ~20 min on 64 cores, then a re-run of an unchanged tree skips this step"
  echo "    the compile runs in step 7: uv prints every compiler line, one status line every 60s"

  # Install the tree in place. setup.py does the cmake configure, compiles this tree's kernels,
  # and drops the products into $SRC/vllm/. --no-deps keeps the pins from step 4,
  # --no-build-isolation compiles against the torch in the venv.
  step 7 "install the tree into .venv (editable, compiled)"
  build_log="$ROOT/build-$(date +%Y%m%d-%H%M%S).log"
  echo "    full build output -> $build_log"
  stats(){ "$cc" -v -s 2>/dev/null | grep -E 'Hits:|Misses:|Cache size' | tr -s ' ' | tr '\n' ' '; }
  before_stats="$(stats || true)"
  # -v is what makes the compile visible. Without it uv swallows the build backend's stdout, so a
  # 20 minute compile prints nothing between "Building vllm @ file:..." and "Built vllm".
  # -v streams every nvcc/cc/cargo line (uv prefixes them with DEBUG). -vv adds uv's own debug noise.
  t0=$(date +%s)
  # Products to watch for, in the order setup.py produces them. Counted from the list, so the
  # heartbeat ends at 15/15 and never counts a file the list does not require.
  GCC_VER=""
  for v in 14 13 12 11 10; do
    if command -v "gcc-$v" >/dev/null 2>&1; then
      GCC_VER="$v"
      break
    fi
  done
  [ -n "$GCC_VER" ] || { echo "Error: No GCC version < 15 found" >&2; exit 1; }
  CC_PATH="/usr/bin/gcc-$GCC_VER"
  CXX_PATH="/usr/bin/g++-$GCC_VER"
  prod=$(grep -v '^\s*#\|^\s*$' "$LIST" | sed "s#^#$SRC/#")
  ( while sleep 60; do
      e=$(( ( $(date +%s) - t0 ) / 60 ))
      p=$(grep -oE '\[[0-9]+/[0-9]+\]' "$build_log" 2>/dev/null | tail -1)
      n=0; while read -r f; do [ -e "$f" ] && n=$(( n + 1 )); done <<< "$prod"
      echo "    ... ${e} min elapsed, cmake ${p:-configuring}, ${n}/$(printf '%s\n' "$prod" | wc -l) build products in place"
    done ) & hb=$!
  ( cd "$SRC" && env CUDA_HOME="$CU" PATH="$(dirname "$NVCC"):$PATH" TORCH_CUDA_ARCH_LIST="$ARCH" \
      MAX_JOBS="$JOBS" VLLM_TARGET_DEVICE=cuda VLLM_VERSION_OVERRIDE=0.30.0 \
      CC="$CC_PATH" CXX="$CXX_PATH" \
      CMAKE_ARGS="-DCMAKE_CUDA_HOST_COMPILER=$CXX_PATH -DCMAKE_C_COMPILER=$CC_PATH -DCMAKE_CXX_COMPILER=$CC_PATH" \
      sh -c "$PIP_INSTALL -v -e . --no-deps --no-build-isolation" ) 2>&1 | tee "$build_log" \
    || { kill "$hb" 2>/dev/null; echo "step 7 failed; the compiler output is in $build_log"; exit 1; }
  kill "$hb" 2>/dev/null || true; wait "$hb" 2>/dev/null || true
  grep -iE "warning: |error: " "$build_log" | grep -v "^\s*$" | head -20 || true
  if [ -n "$cc" ]; then
    echo "    ccache before this build: ${before_stats:-no stats yet}"
    echo "    ccache after this build:  $(stats || true)"
  fi
  check_list || { echo "installed, but build products are missing; see $build_log"; exit 1; }
  write_sums
fi
echo "    $(wc -l < "$SUMS") build files verified ($(du -sh "$SRC/vllm/_C_stable_libtorch.abi3.so" | cut -f1) + $(du -sh "$SRC/vllm/_moe_C_stable_libtorch.abi3.so" | cut -f1) extensions)"

# Smoke test. Importing a compiled kernel loads libcuda.so.1, the CUDA driver
# library. A box with a GPU has it (the driver package); a build container does
# not, so the import cannot run there and does not need to. Without a driver,
# check the link layer instead: the products exist (step 6 hashed them) and the
# loader entry that needs the driver is the one we expect. The full import runs
# at serve time, on a box with the driver.
if ldconfig -p 2>/dev/null | grep -q 'libcuda\.so\.1' || [ -e /dev/nvidia0 ]; then
  smoke="$(cd / && "$VENV/bin/python" -c '
import torch, vllm
import vllm._C_stable_libtorch, vllm._moe_C_stable_libtorch
import vllm.vllm_flash_attn._vllm_fa2_C, vllm.vllm_flash_attn._vllm_fa3_C
from vllm.tool_parsers import ToolParserManager
ToolParserManager.get_tool_parser("qwen3_coder")
print("BUILD_SMOKE", vllm.__version__, vllm.__file__)' | tee /dev/stderr)" \
    || { echo "the installed tree does not import; see $build_log"; exit 1; }
  # vLLM writes its log lines to stdout too: a box with mixed GPUs (3090 + 3090 Ti) prints a WARNING
  # there before the version. Read the version from the marker line, never from the first word.
  line="$(printf '%s\n' "$smoke" | grep '^BUILD_SMOKE ' | tail -1)"
  ver="${line#BUILD_SMOKE }"; ver="${ver%% *}"
  [ "$ver" = 0.30.0 ] || { echo "the installed tree does not report 0.30.0, it printed:"; printf '%s\n' "$smoke" | tail -3 | sed 's/^/      /'; exit 1; }
  echo "    import check: vllm $ver  $(printf '%s\n' "$line" | cut -d' ' -f3-)"
else
  echo "    no CUDA driver here: the import check runs at serve time, on the GPU box"
  if command -v readelf >/dev/null; then
    # Only the extension that calls the driver API directly needs libcuda.so.1 in its NEEDED list
    # (CUTLASS_ENABLE_DIRECT_CUDA_DRIVER_CALL). The MoE kernels do not: they get CUDA through
    # libtorch_cuda/libcudart, so libcuda is absent from their NEEDED list and that is correct.
    # Requiring it there fails a good build (it did, in Docker). Check it where it is expected,
    # and check the torch/CUDA runtime libs everywhere.
    for so in vllm/_C_stable_libtorch.abi3.so vllm/_moe_C_stable_libtorch.abi3.so \
              vllm/vllm_flash_attn/_vllm_fa2_C.abi3.so vllm/vllm_flash_attn/_vllm_fa3_C.abi3.so; do
      [ -e "$SRC/$so" ] || continue
      n="$(readelf -d "$SRC/$so" | sed -n 's/.*\[\(lib[^]]*\)\].*/\1/p' | tr '\n' ' ')"
      [ -n "$n" ] || { echo "$so has no dynamic section; the file is not a shared object"; exit 1; }
      for need in libtorch.so libtorch_cuda.so libcudart.so; do
        case " $n" in *" $need"*) ;; *) echo "$so does not ask for $need; the link is wrong"; exit 1 ;; esac
      done
      case "$so" in
        vllm/_C_stable_libtorch.abi3.so) case " $n" in *libcuda.so*) ;; *) echo "$so does not ask for libcuda.so.1; the link is wrong"; exit 1 ;; esac ;;
      esac
      echo "    link check $(basename "$so"):${n:+ }$n"
    done
    echo "    link check: every lib above is found at run time (libcuda.so.1 comes from the driver)"
  else
    echo "    link check skipped: no readelf on this box"
  fi
fi

{ echo "source $(sha256sum "$SRC/vllm/_C_stable_libtorch.abi3.so" | cut -d' ' -f1) $(sha256sum "$SRC/vllm/_moe_C_stable_libtorch.abi3.so" | cut -d' ' -f1) $TREE_HEAD"; } > "$MARK"

# step 8: pack the kernels for the image build. The same tree object is what step 6 accepts back,
# so this tarball is the fast route for Docker: ARTIFACTS=artifacts/<name>.tar.gz, no compile.
# The tag carries what the products were built for: sm86 (arch 8.6) + the CPython minor + the CUDA
# major. A 3090 tarball does not work on a 5090; CUDA 13 products do not work with CUDA 12.
# An artifacts set is current when the tarball, its per-file list and its manifest are all there, the
# manifest names this tree, and the tarball still matches its own sha256 sidecar. Delete any of them
# (or the whole folder) and this step writes the set again from the products already in the tree, so a
# lost artifacts/ costs a pack (~45s), never a recompile.
artifacts_current(){ local d="$1" n="$2" o
  { [ -s "$d/$n.tar.gz" ] && [ -s "$d/$n.sha256" ] && [ -s "$d/$n.manifest" ] && [ -s "$d/$n.tar.gz.sha256" ]; } || return 1
  [ "$(sed -n 's/^tree_obj=//p' "$d/$n.manifest" | head -1)" = "$TREE_OBJ" ] || return 1
  o="$(sed -n 's/^ops_obj=//p' "$d/$n.manifest" | head -1)"; [ -z "$o" ] || [ "$o" = "$OPS_OBJ" ] || return 1
  ( cd "$d" && sha256sum -c --quiet "$n.tar.gz.sha256" ) ; }
if [ "$PACK" = 0 ]; then
  step 8 "pack the kernels for the image build (skipped: PACK=0 / --no-pack)"
else
  step 8 "pack the kernels for the image build"
  OUT="${OUT:-$HERE/artifacts}"; mkdir -p "$OUT"
  pyv="$("$VENV/bin/python" -c 'import sys; print("%d.%d" % sys.version_info[:2])')"
  tv="$("$VENV/bin/python" -c 'import torch; print(torch.__version__.split("+")[0])')"
  tag="sm${ARCH/./}-py${pyv/./}-cu${nv_rel%%.*}"
  name="build-artifacts-$tag"
  if artifacts_current "$OUT" "$name"; then
    echo "    artifacts/$name.tar.gz already holds tree ${TREE_OBJ:0:7}: not packed again"
  else
    [ "$prebuilt" = 1 ] && echo "    these kernels came from a tarball, and artifacts/ holds no set for this tree: packing them back"
    files="$( cd "$SRC" && while read -r p; do case "$p" in ''|'#'*) continue ;; esac
             find "${p%/}" -type f; done < "$LIST" | LC_ALL=C sort )"
    n="$(printf '%s\n' "$files" | grep -c . || true)"
    [ "$n" -gt 0 ] || { echo "$(basename "$LIST") names no file under $SRC"; exit 1; }
    echo "    packing $n files -> artifacts/$name.tar.gz (~45s)"
    # tar runs inside $SRC so every member starts with vllm/, which is the layout step 6 unpacks.
    ( cd "$SRC" && printf '%s\n' "$files" | xargs -d '\n' tar -czf "$OUT/$name.tar.gz" -- ) \
      || { echo "packing failed"; exit 1; }
    ( cd "$SRC" && printf '%s\n' "$files" | LC_ALL=C sort | xargs -d '\n' sha256sum ) > "$OUT/$name.sha256"
    {
      echo "tree=$TREE_HEAD"
      echo "tree_obj=$TREE_OBJ"
      echo "ops_obj=$OPS_OBJ"
      echo "tree_full=$(git -C "$SRC" rev-parse HEAD)"
      echo "arch=$ARCH"
      echo "python=$pyv"
      echo "cuda=$nv_rel"
      echo "torch=$tv"
      echo "cc=$(cc --version 2>/dev/null | head -1 || echo unknown)"
      echo "glibc=$(glibc_ver || echo unknown)"
      echo "built=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
      echo "files=$n"
      echo "sha_C=$(sha256sum "$SRC/vllm/_C_stable_libtorch.abi3.so" | cut -d' ' -f1)"
      echo "sha_moe=$(sha256sum "$SRC/vllm/_moe_C_stable_libtorch.abi3.so" | cut -d' ' -f1)"
    } > "$OUT/$name.manifest"
    sha256sum "$OUT/$name.tar.gz" | sed "s|  .*$|  $name.tar.gz|" > "$OUT/$name.tar.gz.sha256"
    ls -l "$OUT/$name.tar.gz" "$OUT/$name.sha256" "$OUT/$name.manifest" | sed 's/^/    /'
  fi
fi

echo "done: TREE=$SRC (tree $TREE_HEAD) VENV=$VENV"
echo "next: ./start.sh /path/to/Qwen3.8-Flash-Next-W4A16-Merlin"
echo "      docker: docker build --build-context ops=\"$HERE/artifacts\" -t vllm:latest .  (no compile)"