diff --git a/start.sh b/start.sh index d1eaf80..0cb179b 100755 --- a/start.sh +++ b/start.sh @@ -258,18 +258,19 @@ exec ${topo_launch_prefix:+$topo_launch_prefix} "$VENV/bin/vllm" serve "$CKPT" \ --host "${HOST:-127.0.0.1}" --port "${PORT:-8000}" \ --tensor-parallel-size 2 --pipeline-parallel-size 2 --enable-expert-parallel \ --speculative-config '{"method":"mtp","num_speculative_tokens":3,"draft_sample_method":"probabilistic"}' \ - --max-model-len 262144 --max-num-seqs 2 --max-num-batched-tokens 4096 \ + --max-model-len 262144 --max-num-seqs 4 --max-num-batched-tokens 2048 \ --gpu-memory-utilization 0.96 --kv-cache-memory 4100000000 --kv-cache-dtype fp8_e4m3 \ --limit-mm-per-prompt '{"image":42,"video":0}' --mm-processor-cache-gb 0 \ --mm-processor-kwargs '{"size":{"longest_edge":4194304,"shortest_edge":65536}}' \ - --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,6,8,12,16,24,32]}' \ + --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,4,8,12,16]}' \ --enable-prefix-caching --enable-prompt-tokens-details \ --no-enable-flashinfer-autotune \ --enable-auto-tool-choice --tool-call-parser qwen3_coder --reasoning-parser qwen3 \ --override-generation-config '{"temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0}' \ - --default-chat-template-kwargs '{"enable_thinking": true, "reasoning_effort": "low"}' \ + --default-chat-template-kwargs '{"enable_thinking": true, "reasoning_effort": "xhigh"}' \ --shutdown-timeout 10 \ - --block-size 4096 --use-replayssm \ + --block-size 2048 --use-replayssm \ --prefix-match-unit 64 \ --safetensors-load-strategy prefetch \ + --disable-uvicorn-access-log \ "$@"