MOVZXandHinata
fe3cd9247b
git am refreshes every file timestamp, so a Python-only patch series forced a full extension rebuild and ninja saw everything as changed. Diff csrc/ and the build files between the recorded tree and HEAD, and rebuild only when a C++/CUDA source moved; otherwise the shipped .so stays. Update the expected tree to the 106-patch series (c9c526755364d44d8d9923d9acc9d6d1a3d58718). Add ipc: host, SYS_PTRACE, and seccomp=unconfined to the container so a profiler or debugger can attach. Raise --block-size from 2048 to 4096 and drop --safetensors-load-strategy prefetch. Co-Authored-By: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
70 lines
2.3 KiB
YAML
70 lines
2.3 KiB
YAML
services:
|
|
vllm:
|
|
image: ${IMAGE:-vllm}:${IMAGE_TAG:-latest}
|
|
container_name: ${CONTAINER_NAME:-vllm}
|
|
build:
|
|
context: .
|
|
args:
|
|
MAX_JOBS: ${MAX_JOBS:-32}
|
|
ARCH: ${ARCH:-8.6}
|
|
additional_contexts:
|
|
ops: ./artifacts
|
|
uvcache: ${UV_CACHE_DIR:-${HOME}/.cache/uv}
|
|
pipcache: ${PIP_CACHE_DIR:-${HOME}/.cache/pip}
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices:
|
|
- driver: nvidia
|
|
count: all
|
|
capabilities: [gpu]
|
|
command: ["${MODEL_MOUNT_PATH:-/models/Qwen3.8-Flash-Next}"]
|
|
ipc: host
|
|
shm_size: 16g
|
|
cap_add:
|
|
- SYS_PTRACE
|
|
security_opt:
|
|
- seccomp=unconfined
|
|
ulimits:
|
|
memlock: -1
|
|
stack: 67108864
|
|
ports:
|
|
- "${HOST_BIND_ADDR:-0.0.0.0}:${HOST_PORT:-8080}:8000"
|
|
environment:
|
|
MODEL_NAME: ${MODEL_NAME:-Hinata}
|
|
PORT: 8000
|
|
SCALES: /opt/vllm/qsa_kv_scales.json
|
|
VLLM_API_KEY: ${VLLM_API_KEY:-}
|
|
PLE_HOME: ${PLE_HOME:-0}
|
|
CUDA_VISIBLE_DEVICES: ${CUDA_VISIBLE_DEVICES:-0,1,2,3}
|
|
AUTO_TOPO: ${AUTO_TOPO:-0}
|
|
AUTO_TOPO_NCCL: ${AUTO_TOPO_NCCL:-legacy}
|
|
AUTO_TOPO_BIND: ${AUTO_TOPO_BIND:-0}
|
|
LANE_GPUS: ${LANE_GPUS:-}
|
|
LANE_CPUS: ${LANE_CPUS:-}
|
|
TOPO_TIMEOUT: ${TOPO_TIMEOUT:-20}
|
|
GUARD: ${GUARD:-warn}
|
|
COUNTERS: ${COUNTERS:-0}
|
|
VLLM_LOG_STATS_INTERVAL: ${VLLM_LOG_STATS_INTERVAL:-1}
|
|
MAX_SEQS: ${MAX_SEQS:-4}
|
|
MAX_BATCHED_TOKENS: ${MAX_BATCHED_TOKENS:-2048}
|
|
QWEN38_ABLITERATION: ${QWEN38_ABLITERATION:-Qwen3.8-Flash-Next-Uncensored}
|
|
volumes:
|
|
- "${MODEL_DIR:?Set MODEL_DIR in .env}:${MODEL_MOUNT_PATH:-/models/Qwen3.8-Flash-Next}:ro"
|
|
- ${ABLITERATION_DIR:-./abliteration}:/opt/qwen38/abliteration:ro
|
|
- ${CACHE_HOST_DIR:-./.vllm-cache}:/cache
|
|
- ${SCALES:-${MODEL_DIR}/qsa_kv_scales.json}:/opt/vllm/qsa_kv_scales.json:ro
|
|
- ./start.sh:/opt/vllm/start.sh:ro
|
|
restart: unless-stopped
|
|
stop_grace_period: 20s
|
|
logging:
|
|
driver: json-file
|
|
options:
|
|
max-size: ${LOG_MAX_SIZE:-100m}
|
|
max-file: ${LOG_MAX_FILE:-14}
|
|
healthcheck:
|
|
interval: ${HEALTHCHECK_INTERVAL:-60s}
|
|
timeout: ${HEALTHCHECK_TIMEOUT:-90s}
|
|
start_period: ${HEALTHCHECK_START_PERIOD:-30m}
|
|
retries: ${HEALTHCHECK_RETRIES:-3}
|