diff --git a/Dockerfile b/Dockerfile index 5eed3ce..0967c8f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,7 +35,7 @@ FROM python:3.13-slim-trixie # -v "$PWD/.vllm-cache":/cache \ # -e MODEL_NAME=Hinata vllm:latest # -# The checkpoint mount must contain qsa_kv_scales_swift.json (the FP8 KV scales sidecar). +# The checkpoint mount must contain qsa_kv_scales.json (the FP8 KV scales sidecar). # # The build fetches vLLM v0.30.0 and applies patches/ (build.sh step 2; needs network), then compiles # every kernel of this tree from source (build.sh: cmake + ninja + nvcc 13.0 from the venv's diff --git a/README-ID.md b/README-ID.md index 2d64683..6809fe3 100644 --- a/README-ID.md +++ b/README-ID.md @@ -20,7 +20,7 @@ satu aliran diukur pada 158-184 tok/s. | Python | 3.13 | | Build tools | `cmake` 3.26 atau lebih baru, `ninja`, `cc`, `c++` | -Checkpoint harus berisi `qsa_kv_scales_swift.json`. File itu menyimpan scale KV +Checkpoint harus berisi `qsa_kv_scales.json`. File itu menyimpan scale KV FP8. Swift 1.5 mengubah `k_proj`/`v_proj`, jadi file scale satu checkpoint tidak bekerja untuk checkpoint lain. diff --git a/README.md b/README.md index 067bb0e..d271e33 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ differ, this file wins. | Python | 3.13 | | Build tools | `cmake` 3.26 or newer, `ninja`, `cc`, `c++` | -The checkpoint must contain `qsa_kv_scales_swift.json`. That file holds the FP8 KV +The checkpoint must contain `qsa_kv_scales.json`. That file holds the FP8 KV scales. Swift 1.5 changed `k_proj`/`v_proj`, so the scale file of one checkpoint does not work for another. diff --git a/start.sh b/start.sh index 9fe487c..756e3d7 100755 --- a/start.sh +++ b/start.sh @@ -17,7 +17,7 @@ # `-` is a vllm argument, so the default checkpoint is kept. # TREE VENV serving tree and venv (default: ./vllm and ./.venv, built by build.sh) # PORT (8000) HOST (127.0.0.1) MODEL_NAME ("name1 name2 ...", first is the health probe's) -# SCALES KV scale sidecar (default: qsa_kv_scales_swift.json inside the checkpoint) +# SCALES KV scale sidecar, host path (default: $CKPT/qsa_kv_scales.json) # CACHE_ROOT compile/cudagraph cache (default: .vllm-cache next to this script, on disk) # PLE_HOME GPU that hosts the PLE offload home (0) # GUARD bounds-guard telemetry off|warn|raise (warn) @@ -201,7 +201,7 @@ export VLLM_E2_PLE_PULL_TRANSPORT=1 VLLM_PLE_OFFLOAD_READY_TIMEOUT=1800 export VLLM_PLE_OFFLOAD_HOME_DEVICE="${PLE_HOME:-0}" export VLLM_HOST_EMBED_TABLE=1 export VLLM_USE_BREAKABLE_CUDAGRAPH=0 -export VLLM_QSA_KV_SCALES="${SCALES:-$CKPT/qsa_kv_scales_swift.json}" +export VLLM_QSA_KV_SCALES="${SCALES:-$CKPT/qsa_kv_scales.json}" # engine stats line (throughput, KV usage, queue). Set empty to keep vLLM's default. export HF_HUB_OFFLINE=1 HF_DATASETS_OFFLINE=1 # the kernel toggles (the shipped values). These names are the tree's own: envs.py in the @@ -210,7 +210,7 @@ export MERLIN_FULL_K=1 MERLIN_QSA_SORT=1 MERLIN_TIE_DET=1 MERLIN_TIE_RECENT=1 ME export VLLM_E2_GUARD_MODE="${GUARD:-warn}" export VLLM_E2_SIDE_CACHE_LAYOUT=1 VLLM_USE_V2_MODEL_RUNNER=1 VLLM_E2_MAMBA_RETAIN_CHECKPOINT=1 if [ "${COUNTERS:-0}" = 1 ]; then export MERLIN_TOPK_EXACT_COUNTERS_LOG=30 VLLM_QSA_KV_CLIP_COUNT=300; fi -[ -f "$VLLM_QSA_KV_SCALES" ] || { echo "scales sidecar missing: $VLLM_QSA_KV_SCALES (FP8 KV would run at scale 1.0)"; exit 1; } +[ -f "$VLLM_QSA_KV_SCALES" ] || { echo "scales sidecar missing: $VLLM_QSA_KV_SCALES (FP8 KV would run at scale 1.0). Set SCALES= to the sidecar of this model."; exit 1; } # Compile cache: on disk by default, so it survives a reboot. A cold cache costs ~6 min of kernel # compile, and the cache is only ~160 MB, so disk is the right home for it. Keep ONE root per