MOVZXandHinata
b89e85e99f
The container read the sidecar only inside the MODEL_DIR mount, so a scale file in another folder needed a manual -v and a container path. Bind-mount the file named by SCALES to /opt/vllm/qsa_kv_scales.json, and pass that fixed path to the engine. The default stays $MODEL_DIR/qsa_kv_scales.json, so the shipped setup does not change. Add a commented SCALES entry to .env.example. Co-Authored-By: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
62 lines
2.0 KiB
YAML
62 lines
2.0 KiB
YAML
services:
|
|
vllm:
|
|
image: ${IMAGE:-vllm}:${IMAGE_TAG:-latest}
|
|
container_name: ${CONTAINER_NAME:-vllm}
|
|
build:
|
|
context: .
|
|
args:
|
|
MAX_JOBS: ${MAX_JOBS:-32}
|
|
ARCH: ${ARCH:-8.6}
|
|
additional_contexts:
|
|
ops: ./artifacts
|
|
uvcache: ${UV_CACHE_DIR:-${HOME}/.cache/uv}
|
|
pipcache: ${PIP_CACHE_DIR:-${HOME}/.cache/pip}
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices:
|
|
- driver: nvidia
|
|
count: all
|
|
capabilities: [gpu]
|
|
command: ["${MODEL_MOUNT_PATH:-/models/Qwen3.8-Flash-Next}"]
|
|
shm_size: 16g
|
|
ulimits:
|
|
memlock: -1
|
|
stack: 67108864
|
|
ports:
|
|
- "${HOST_BIND_ADDR:-0.0.0.0}:${HOST_PORT:-8080}:8000"
|
|
environment:
|
|
MODEL_NAME: ${MODEL_NAME:-Hinata}
|
|
PORT: 8000
|
|
SCALES: /opt/vllm/qsa_kv_scales.json
|
|
VLLM_API_KEY: ${VLLM_API_KEY:-}
|
|
PLE_HOME: ${PLE_HOME:-0}
|
|
CUDA_VISIBLE_DEVICES: ${CUDA_VISIBLE_DEVICES:-0,1,2,3}
|
|
AUTO_TOPO: ${AUTO_TOPO:-0}
|
|
AUTO_TOPO_NCCL: ${AUTO_TOPO_NCCL:-legacy}
|
|
AUTO_TOPO_BIND: ${AUTO_TOPO_BIND:-0}
|
|
LANE_GPUS: ${LANE_GPUS:-}
|
|
LANE_CPUS: ${LANE_CPUS:-}
|
|
TOPO_TIMEOUT: ${TOPO_TIMEOUT:-20}
|
|
GUARD: ${GUARD:-warn}
|
|
COUNTERS: ${COUNTERS:-0}
|
|
QWEN38_ABLITERATION: ${QWEN38_ABLITERATION:-Qwen3.8-Flash-Next-Uncensored}
|
|
volumes:
|
|
- "${MODEL_DIR:?Set MODEL_DIR in .env}:${MODEL_MOUNT_PATH:-/models/Qwen3.8-Flash-Next}:ro"
|
|
- ${ABLITERATION_DIR:-./abliteration}:/opt/qwen38/abliteration:ro
|
|
- ${CACHE_HOST_DIR:-./.vllm-cache}:/cache
|
|
- ${SCALES:-${MODEL_DIR}/qsa_kv_scales.json}:/opt/vllm/qsa_kv_scales.json:ro
|
|
- ./start.sh:/opt/vllm/start.sh:ro
|
|
restart: unless-stopped
|
|
stop_grace_period: 20s
|
|
logging:
|
|
driver: json-file
|
|
options:
|
|
max-size: ${LOG_MAX_SIZE:-100m}
|
|
max-file: ${LOG_MAX_FILE:-14}
|
|
healthcheck:
|
|
interval: ${HEALTHCHECK_INTERVAL:-60s}
|
|
timeout: ${HEALTHCHECK_TIMEOUT:-90s}
|
|
start_period: ${HEALTHCHECK_START_PERIOD:-30m}
|
|
retries: ${HEALTHCHECK_RETRIES:-3}
|