Add script for benchmarking serving throughput (#145)

This commit is contained in:
Woosuk Kwon
2023-06-14 19:55:38 -07:00
committed by GitHub
parent da5ddcd544
commit 311490a720
10 changed files with 421 additions and 415 deletions
+16
View File
@@ -0,0 +1,16 @@
#!/bin/bash
PORT=8001
MODEL=$1
TOKENS=$2
docker run --gpus all --shm-size 1g -p $PORT:80 \
-v $PWD/data:/data \
ghcr.io/huggingface/text-generation-inference:0.8 \
--model-id $MODEL \
--sharded false \
--max-input-length 1024 \
--max-total-tokens 2048 \
--max-best-of 5 \
--max-concurrent-requests 5000 \
--max-batch-total-tokens $TOKENS