diff --git a/.env.example b/.env.example index a76abbf..9f58b1f 100644 --- a/.env.example +++ b/.env.example @@ -33,6 +33,12 @@ TOPO_TIMEOUT=20 GUARD=warn # 1 adds the two logging-only counters (exact-restart counters /30 s, FP8 KV clips /300 s). COUNTERS=0 +# Concurrent requests. CUDA graphs are captured up to MAX_SEQS x 4 tokens (MTP K=3), so a raise +# adds capture time and graph VRAM. N x 262144 tokens above the 924,993-token KV pool preempts. +MAX_SEQS=4 +# Prefill chunk in tokens. Must be >= MAX_SEQS. 2048 matches --block-size; 4096 prefills one +# max-size image in one chunk. +MAX_BATCHED_TOKENS=2048 # Seconds between engine stats lines (throughput, KV usage, queue depth). The vLLM default is 10 s. VLLM_LOG_STATS_INTERVAL=1 diff --git a/README-ID.md b/README-ID.md index d3513d5..bd0dd3f 100644 --- a/README-ID.md +++ b/README-ID.md @@ -320,6 +320,8 @@ ke seri dan ekspor ulang dengan `git format-patch`. | `GUARD` | `warn` | Telemetri bounds-guard: `off`, `warn`, atau `raise`. | | `COUNTERS` | `0` | `1` menambah counter kernel khusus log. | | `VLLM_LOG_STATS_INTERVAL` | `1` | Jeda antar baris stats engine, dalam detik. Default vLLM: 10 s. | +| `MAX_SEQS` | `4` | Jumlah request serentak. CUDA graph dicapture sampai batch `MAX_SEQS` x 4 token. | +| `MAX_BATCHED_TOKENS` | `2048` | Ukuran chunk prefill dalam token. Harus >= `MAX_SEQS`. Sama dengan `--block-size`. | | `MAX_JOBS`, `ARCH` | `32`, `8.6` | Job compile dan arsitektur GPU, hanya untuk build. | ## 10. Benchmark diff --git a/README.md b/README.md index 1cd64ef..5aee5ea 100644 --- a/README.md +++ b/README.md @@ -309,6 +309,8 @@ onto the series and re-export it with `git format-patch`. | `GUARD` | `warn` | Bounds-guard telemetry: `off`, `warn`, or `raise`. | | `COUNTERS` | `0` | `1` adds the logging-only kernel counters. | | `VLLM_LOG_STATS_INTERVAL` | `1` | Seconds between engine stats lines. The vLLM default is 10 s. | +| `MAX_SEQS` | `4` | Concurrent requests. CUDA graphs cover batches up to `MAX_SEQS` x 4 tokens. | +| `MAX_BATCHED_TOKENS` | `2048` | Prefill chunk in tokens. Must be >= `MAX_SEQS`. 2048 matches `--block-size`. | | `MAX_JOBS`, `ARCH` | `32`, `8.6` | Compile jobs and GPU architecture, for the build only. | ## 10. Benchmarks diff --git a/docker-compose.yml b/docker-compose.yml index d754687..7384704 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -41,6 +41,8 @@ services: GUARD: ${GUARD:-warn} COUNTERS: ${COUNTERS:-0} VLLM_LOG_STATS_INTERVAL: ${VLLM_LOG_STATS_INTERVAL:-1} + MAX_SEQS: ${MAX_SEQS:-4} + MAX_BATCHED_TOKENS: ${MAX_BATCHED_TOKENS:-2048} QWEN38_ABLITERATION: ${QWEN38_ABLITERATION:-Qwen3.8-Flash-Next-Uncensored} volumes: - "${MODEL_DIR:?Set MODEL_DIR in .env}:${MODEL_MOUNT_PATH:-/models/Qwen3.8-Flash-Next}:ro" diff --git a/start.sh b/start.sh index d58492c..9ef29b0 100755 --- a/start.sh +++ b/start.sh @@ -23,6 +23,10 @@ # GUARD bounds-guard telemetry off|warn|raise (warn) # COUNTERS=1 logging-only counters (top-k exact restarts /30s, FP8 KV clips /300s) # VLLM_LOG_STATS_INTERVAL seconds between engine stats lines (1; the vLLM default is 10 s) +# MAX_SEQS (4) concurrent requests. The CUDA graph list follows it, to MAX_SEQS x 4 +# tokens (MTP K=3). N x 262144 tokens above the 924,993-token pool preempts. +# MAX_BATCHED_TOKENS (2048) prefill chunk in tokens. Must be >= MAX_SEQS. 2048 matches +# --block-size; 4096 prefills one max-size image in one chunk. # CUDA_HOME toolkit for first-serve JIT; default: the venv's CUDA 13.0 wheels # AUTO_TOPO 0 (default, fixed shipped settings) | 1 | strict -> scripts/topo_select.py # AUTO_TOPO_NCCL=default AUTO_TOPO_BIND=1 LANE_GPUS=auto LANE_CPUS=... TOPO_TIMEOUT=20 @@ -219,6 +223,40 @@ export VLLM_SKIP_MODEL_NAME_VALIDATION="${VLLM_SKIP_MODEL_NAME_VALIDATION:-1}" if [ "${COUNTERS:-0}" = 1 ]; then export MERLIN_TOPK_EXACT_COUNTERS_LOG=30 VLLM_QSA_KV_CLIP_COUNT=300; fi [ -f "$VLLM_QSA_KV_SCALES" ] || { echo "scales sidecar missing: $VLLM_QSA_KV_SCALES (FP8 KV would run at scale 1.0). Set SCALES= to the sidecar of this model."; exit 1; } +# Concurrency and the prefill chunk, tunable from .env. vLLM refuses a chunk smaller than the +# sequence count (vllm/config/scheduler.py:309); say it here, where the reader set the value. +MAX_SEQS="${MAX_SEQS:-4}" +MAX_BATCHED_TOKENS="${MAX_BATCHED_TOKENS:-2048}" +for v in MAX_SEQS MAX_BATCHED_TOKENS; do + case "${!v}" in ''|*[!0-9]*) echo "$v must be a positive integer (got: ${!v})"; exit 1 ;; esac + [ "${!v}" -ge 1 ] || { echo "$v must be 1 or more (got: ${!v})"; exit 1; } +done +if [ "$MAX_BATCHED_TOKENS" -lt "$MAX_SEQS" ]; then + echo "MAX_BATCHED_TOKENS ($MAX_BATCHED_TOKENS) must be >= MAX_SEQS ($MAX_SEQS)"; exit 1 +fi +# The shipped pool holds 924,993 tokens, pinned by --kv-cache-memory. A shape that needs more does +# not fail: the engine preempts and recomputes. +if [ $((MAX_SEQS * 262144)) -gt 924993 ]; then + echo "note: MAX_SEQS=$MAX_SEQS wants $((MAX_SEQS * 262144)) tokens at the full 262144 context; the pool holds 924993, so full-length requests preempt" +fi +# CUDA graphs must cover the widest decode batch: MAX_SEQS x (K+1) tokens, K the MTP depth. A +# batch pads up to the next captured size, and runs outside the graph above the largest one. The +# sizes below the MTP width stay for small and mixed batches. Each size costs capture time and +# VRAM, so the list stops at the widest batch, as the shipped shape does (4 x 4 = 16). +# MTP depth: 3 is Qwen 3.8's recommended value, not a knob. The graph math below reads it, so keep +# it equal to the num_speculative_tokens in --speculative-config. +MTP_K=3 +CG_TOP=$((MAX_SEQS * (MTP_K + 1))) +CG_SIZES="1,2,4" +for ((n = MTP_K + 1; n <= CG_TOP; n += MTP_K + 1)); do + [ "$n" -gt 4 ] && CG_SIZES="$CG_SIZES,$n" +done +CG_COUNT=$((2 + MAX_SEQS)) +if [ "$CG_COUNT" -gt 8 ]; then + echo "note: $CG_COUNT CUDA graphs for MAX_SEQS=$MAX_SEQS; capture time and graph VRAM both grow with the count" +fi +echo "cuda graphs: 1,2,4 then every $((MTP_K + 1)) tokens to $CG_TOP" + # Compile cache: on disk by default, so it survives a reboot. A cold cache costs ~6 min of kernel # compile, and the cache is only ~160 MB, so disk is the right home for it. Keep ONE root per # deployment -- a new root costs a fresh compile. CACHE_ROOT=/vllm-cache (a tmpfs) is the opt-in @@ -263,11 +301,11 @@ exec ${topo_launch_prefix:+$topo_launch_prefix} "$VENV/bin/vllm" serve "$CKPT" \ --host "${HOST:-127.0.0.1}" --port "${PORT:-8000}" \ --tensor-parallel-size 2 --pipeline-parallel-size 2 --enable-expert-parallel \ --speculative-config '{"method":"mtp","num_speculative_tokens":3,"draft_sample_method":"probabilistic"}' \ - --max-model-len 262144 --max-num-seqs 4 --max-num-batched-tokens 2048 \ + --max-model-len 262144 --max-num-seqs "$MAX_SEQS" --max-num-batched-tokens "$MAX_BATCHED_TOKENS" \ --gpu-memory-utilization 0.96 --kv-cache-memory 4100000000 --kv-cache-dtype fp8_e4m3 \ - --offload-towers --limit-mm-per-prompt '{"image":42,"video":0}' --mm-processor-cache-gb 16 \ + --offload-towers --limit-mm-per-prompt '{"image":42,"video":0}' \ --mm-processor-kwargs '{"size":{"longest_edge":4194304,"shortest_edge":65536}}' \ - --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,4,8,12,16]}' \ + --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":['"$CG_SIZES"']}' \ --enable-prefix-caching --enable-prompt-tokens-details \ --no-enable-flashinfer-autotune \ --enable-auto-tool-choice --tool-call-parser qwen3_coder --reasoning-parser qwen3 \