From eb8c302bb374d1795072f95e921cbcff464196c2 Mon Sep 17 00:00:00 2001 From: MOVZX Date: Thu, 15 Oct 2026 06:18:06 +0000 Subject: [PATCH] fix: Adjust start script to avoid crashing on high pressure KV cache Stress-tested using 4 concurrent requests with 240K tokens each: ``` (APIServer pid=1) INFO 10-15 06:21:25 [loggers.py:315] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 3.0 tokens/s, Running: 4 reqs, Waiting: 1 reqs, GPU KV cache usage: 98.8%, Prefix cache hit rate: 0.0% (APIServer pid=1) INFO 10-15 06:21:26 [loggers.py:315] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 17.0 tokens/s, Running: 4 reqs, Waiting: 1 reqs, GPU KV cache usage: 99.2%, Prefix cache hit rate: 0.0% (APIServer pid=1) INFO 10-15 06:21:27 [loggers.py:315] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 8.0 tokens/s, Running: 4 reqs, Waiting: 1 reqs, GPU KV cache usage: 99.6%, Prefix cache hit rate: 0.0% (APIServer pid=1) INFO 10-15 06:21:28 [loggers.py:315] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 20.9 tokens/s, Running: 4 reqs, Waiting: 1 reqs, GPU KV cache usage: 100.0%, Prefix cache hit rate: 0.0% ``` Co-Authored-By: Hinata Signed-off-by: MOVZX --- start.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/start.sh b/start.sh index d1eaf80..0cb179b 100755 --- a/start.sh +++ b/start.sh @@ -258,18 +258,19 @@ exec ${topo_launch_prefix:+$topo_launch_prefix} "$VENV/bin/vllm" serve "$CKPT" \ --host "${HOST:-127.0.0.1}" --port "${PORT:-8000}" \ --tensor-parallel-size 2 --pipeline-parallel-size 2 --enable-expert-parallel \ --speculative-config '{"method":"mtp","num_speculative_tokens":3,"draft_sample_method":"probabilistic"}' \ - --max-model-len 262144 --max-num-seqs 2 --max-num-batched-tokens 4096 \ + --max-model-len 262144 --max-num-seqs 4 --max-num-batched-tokens 2048 \ --gpu-memory-utilization 0.96 --kv-cache-memory 4100000000 --kv-cache-dtype fp8_e4m3 \ --limit-mm-per-prompt '{"image":42,"video":0}' --mm-processor-cache-gb 0 \ --mm-processor-kwargs '{"size":{"longest_edge":4194304,"shortest_edge":65536}}' \ - --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,6,8,12,16,24,32]}' \ + --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,4,8,12,16]}' \ --enable-prefix-caching --enable-prompt-tokens-details \ --no-enable-flashinfer-autotune \ --enable-auto-tool-choice --tool-call-parser qwen3_coder --reasoning-parser qwen3 \ --override-generation-config '{"temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0}' \ - --default-chat-template-kwargs '{"enable_thinking": true, "reasoning_effort": "low"}' \ + --default-chat-template-kwargs '{"enable_thinking": true, "reasoning_effort": "xhigh"}' \ --shutdown-timeout 10 \ - --block-size 4096 --use-replayssm \ + --block-size 2048 --use-replayssm \ --prefix-match-unit 64 \ --safetensors-load-strategy prefetch \ + --disable-uvicorn-access-log \ "$@"