Change the name to vLLM (#150)

This commit is contained in:
Woosuk Kwon
2023-06-17 03:07:40 -07:00
committed by GitHub
parent e5464ee484
commit 0b98ba15c7
90 changed files with 342 additions and 339 deletions
+1 -1
View File
@@ -1,4 +1,4 @@
# Benchmarking CacheFlow
# Benchmarking vLLM
## Downloading the ShareGPT dataset
+1 -1
View File
@@ -11,7 +11,7 @@ def main(args: argparse.Namespace):
for i in range(args.n_threads)]
api_url = f"http://{args.host}:{args.port}/generate"
headers = {"User-Agent": "CacheFlow Benchmark Client"}
headers = {"User-Agent": "vLLM Benchmark Client"}
ploads = [{
"prompt": p,
"max_tokens": args.max_tokens,
+1 -1
View File
@@ -6,7 +6,7 @@ import numpy as np
import torch
from tqdm import tqdm
from cacheflow import LLM, SamplingParams
from vllm import LLM, SamplingParams
def main(args: argparse.Namespace):
+5 -5
View File
@@ -1,8 +1,8 @@
"""Benchmark online serving throughput.
On the server side, run one of the following commands:
(CacheFlow backend)
python -m cacheflow.entrypoints.api_server \
(vLLM backend)
python -m vllm.entrypoints.api_server \
--disable-log-requests --model <your_model>
(TGI backend)
@@ -114,7 +114,7 @@ async def send_request(
request_start_time = time.time()
headers = {"User-Agent": "Benchmark Client"}
if backend == "cacheflow":
if backend == "vllm":
pload = {
"prompt": prompt,
"n": 1,
@@ -213,8 +213,8 @@ def main(args: argparse.Namespace):
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Benchmark the online serving throughput.")
parser.add_argument("--backend", type=str, default="cacheflow",
choices=["cacheflow", "tgi"])
parser.add_argument("--backend", type=str, default="vllm",
choices=["vllm", "tgi"])
parser.add_argument("--host", type=str, default="localhost")
parser.add_argument("--port", type=int, default=8001)
parser.add_argument("--dataset", type=str, required=True,
+8 -7
View File
@@ -5,12 +5,13 @@ import random
import time
from typing import List, Tuple
from cacheflow import LLM, SamplingParams
import torch
from transformers import (AutoConfig, AutoTokenizer, AutoModelForCausalLM,
PreTrainedTokenizerBase)
from tqdm import tqdm
from vllm import LLM, SamplingParams
def get_tokenizer(model_name: str) -> PreTrainedTokenizerBase:
config = AutoConfig.from_pretrained(model_name)
@@ -70,7 +71,7 @@ def sample_requests(
return sampled_requests
def run_cacheflow(
def run_vllm(
requests: List[Tuple[str, int, int]],
model: str,
tensor_parallel_size: int,
@@ -172,8 +173,8 @@ def main(args: argparse.Namespace):
tokenizer = get_tokenizer(args.model)
requests = sample_requests(args.dataset, args.num_prompts, tokenizer)
if args.backend == "cacheflow":
elapsed_time = run_cacheflow(
if args.backend == "vllm":
elapsed_time = run_vllm(
requests, args.model, args.tensor_parallel_size, args.seed, args.n,
args.use_beam_search)
elif args.backend == "hf":
@@ -192,8 +193,8 @@ def main(args: argparse.Namespace):
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Benchmark the throughput.")
parser.add_argument("--backend", type=str, choices=["cacheflow", "hf"],
default="cacheflow")
parser.add_argument("--backend", type=str, choices=["vllm", "hf"],
default="vllm")
parser.add_argument("--dataset", type=str, required=True,
help="Path to the dataset.")
parser.add_argument("--model", type=str, default="facebook/opt-125m")
@@ -207,7 +208,7 @@ if __name__ == "__main__":
parser.add_argument("--hf-max-batch-size", type=int, default=None,
help="Maximum batch size for HF backend.")
args = parser.parse_args()
if args.backend == "cacheflow":
if args.backend == "vllm":
if args.hf_max_batch_size is not None:
raise ValueError("HF max batch size is only for HF backend.")
elif args.backend == "hf":