diff --git a/.env.example b/.env.example index 13ae041..8ade668 100644 --- a/.env.example +++ b/.env.example @@ -7,6 +7,8 @@ VLLM_API_KEY= MODEL_DIR=/data/models/halt95/Swift1.5-Qwen3.8-Flash-Next-W4A16-Merlin MODEL_MOUNT_PATH=/models/Qwen3.8-Flash-Next MODEL_NAME=Hinata +# The FP8 KV scale sidecar, a host path. Default: $MODEL_DIR/qsa_kv_scales.json. +# SCALES= # ── image and container ────────────────────────────────────────────────────── IMAGE=qwen38-flash-next diff --git a/docker-compose.yml b/docker-compose.yml index d2433a6..2a18dd2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,7 +28,7 @@ services: environment: MODEL_NAME: ${MODEL_NAME:-Hinata} PORT: 8000 - SCALES: ${SCALES:-} + SCALES: /opt/vllm/qsa_kv_scales.json VLLM_API_KEY: ${VLLM_API_KEY:-} PLE_HOME: ${PLE_HOME:-0} CUDA_VISIBLE_DEVICES: ${CUDA_VISIBLE_DEVICES:-0,1,2,3} @@ -45,6 +45,7 @@ services: - "${MODEL_DIR:?Set MODEL_DIR in .env}:${MODEL_MOUNT_PATH:-/models/Qwen3.8-Flash-Next}:ro" - ${ABLITERATION_DIR:-./abliteration}:/opt/qwen38/abliteration:ro - ${CACHE_HOST_DIR:-./.vllm-cache}:/cache + - ${SCALES:-${MODEL_DIR}/qsa_kv_scales.json}:/opt/vllm/qsa_kv_scales.json:ro - ./start.sh:/opt/vllm/start.sh:ro restart: unless-stopped stop_grace_period: 20s