MOVZXandHinata
3a0a5d1510
The stats line carried throughput, KV usage and queue depth, but the interval was never set. Unset, vLLM waits 10 s (vllm/envs.py:47), so a quiet run looked idle for ten seconds and a short one printed nothing. Export VLLM_LOG_STATS_INTERVAL in start.sh with a default of 1, and pass it through Compose so .env can raise it back to 10. Add the variable to .env.example, to the start.sh header, and to the Variables table in both READMEs. Co-Authored-By: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
63 lines
2.1 KiB
YAML
63 lines
2.1 KiB
YAML
services:
|
|
vllm:
|
|
image: ${IMAGE:-vllm}:${IMAGE_TAG:-latest}
|
|
container_name: ${CONTAINER_NAME:-vllm}
|
|
build:
|
|
context: .
|
|
args:
|
|
MAX_JOBS: ${MAX_JOBS:-32}
|
|
ARCH: ${ARCH:-8.6}
|
|
additional_contexts:
|
|
ops: ./artifacts
|
|
uvcache: ${UV_CACHE_DIR:-${HOME}/.cache/uv}
|
|
pipcache: ${PIP_CACHE_DIR:-${HOME}/.cache/pip}
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices:
|
|
- driver: nvidia
|
|
count: all
|
|
capabilities: [gpu]
|
|
command: ["${MODEL_MOUNT_PATH:-/models/Qwen3.8-Flash-Next}"]
|
|
shm_size: 16g
|
|
ulimits:
|
|
memlock: -1
|
|
stack: 67108864
|
|
ports:
|
|
- "${HOST_BIND_ADDR:-0.0.0.0}:${HOST_PORT:-8080}:8000"
|
|
environment:
|
|
MODEL_NAME: ${MODEL_NAME:-Hinata}
|
|
PORT: 8000
|
|
SCALES: /opt/vllm/qsa_kv_scales.json
|
|
VLLM_API_KEY: ${VLLM_API_KEY:-}
|
|
PLE_HOME: ${PLE_HOME:-0}
|
|
CUDA_VISIBLE_DEVICES: ${CUDA_VISIBLE_DEVICES:-0,1,2,3}
|
|
AUTO_TOPO: ${AUTO_TOPO:-0}
|
|
AUTO_TOPO_NCCL: ${AUTO_TOPO_NCCL:-legacy}
|
|
AUTO_TOPO_BIND: ${AUTO_TOPO_BIND:-0}
|
|
LANE_GPUS: ${LANE_GPUS:-}
|
|
LANE_CPUS: ${LANE_CPUS:-}
|
|
TOPO_TIMEOUT: ${TOPO_TIMEOUT:-20}
|
|
GUARD: ${GUARD:-warn}
|
|
COUNTERS: ${COUNTERS:-0}
|
|
VLLM_LOG_STATS_INTERVAL: ${VLLM_LOG_STATS_INTERVAL:-1}
|
|
QWEN38_ABLITERATION: ${QWEN38_ABLITERATION:-Qwen3.8-Flash-Next-Uncensored}
|
|
volumes:
|
|
- "${MODEL_DIR:?Set MODEL_DIR in .env}:${MODEL_MOUNT_PATH:-/models/Qwen3.8-Flash-Next}:ro"
|
|
- ${ABLITERATION_DIR:-./abliteration}:/opt/qwen38/abliteration:ro
|
|
- ${CACHE_HOST_DIR:-./.vllm-cache}:/cache
|
|
- ${SCALES:-${MODEL_DIR}/qsa_kv_scales.json}:/opt/vllm/qsa_kv_scales.json:ro
|
|
- ./start.sh:/opt/vllm/start.sh:ro
|
|
restart: unless-stopped
|
|
stop_grace_period: 20s
|
|
logging:
|
|
driver: json-file
|
|
options:
|
|
max-size: ${LOG_MAX_SIZE:-100m}
|
|
max-file: ${LOG_MAX_FILE:-14}
|
|
healthcheck:
|
|
interval: ${HEALTHCHECK_INTERVAL:-60s}
|
|
timeout: ${HEALTHCHECK_TIMEOUT:-90s}
|
|
start_period: ${HEALTHCHECK_START_PERIOD:-30m}
|
|
retries: ${HEALTHCHECK_RETRIES:-3}
|