Change the name to vLLM (#150)
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
# Benchmarking CacheFlow
|
||||
# Benchmarking vLLM
|
||||
|
||||
## Downloading the ShareGPT dataset
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ def main(args: argparse.Namespace):
|
||||
for i in range(args.n_threads)]
|
||||
|
||||
api_url = f"http://{args.host}:{args.port}/generate"
|
||||
headers = {"User-Agent": "CacheFlow Benchmark Client"}
|
||||
headers = {"User-Agent": "vLLM Benchmark Client"}
|
||||
ploads = [{
|
||||
"prompt": p,
|
||||
"max_tokens": args.max_tokens,
|
||||
|
||||
@@ -6,7 +6,7 @@ import numpy as np
|
||||
import torch
|
||||
from tqdm import tqdm
|
||||
|
||||
from cacheflow import LLM, SamplingParams
|
||||
from vllm import LLM, SamplingParams
|
||||
|
||||
|
||||
def main(args: argparse.Namespace):
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
"""Benchmark online serving throughput.
|
||||
|
||||
On the server side, run one of the following commands:
|
||||
(CacheFlow backend)
|
||||
python -m cacheflow.entrypoints.api_server \
|
||||
(vLLM backend)
|
||||
python -m vllm.entrypoints.api_server \
|
||||
--disable-log-requests --model <your_model>
|
||||
|
||||
(TGI backend)
|
||||
@@ -114,7 +114,7 @@ async def send_request(
|
||||
request_start_time = time.time()
|
||||
|
||||
headers = {"User-Agent": "Benchmark Client"}
|
||||
if backend == "cacheflow":
|
||||
if backend == "vllm":
|
||||
pload = {
|
||||
"prompt": prompt,
|
||||
"n": 1,
|
||||
@@ -213,8 +213,8 @@ def main(args: argparse.Namespace):
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Benchmark the online serving throughput.")
|
||||
parser.add_argument("--backend", type=str, default="cacheflow",
|
||||
choices=["cacheflow", "tgi"])
|
||||
parser.add_argument("--backend", type=str, default="vllm",
|
||||
choices=["vllm", "tgi"])
|
||||
parser.add_argument("--host", type=str, default="localhost")
|
||||
parser.add_argument("--port", type=int, default=8001)
|
||||
parser.add_argument("--dataset", type=str, required=True,
|
||||
|
||||
@@ -5,12 +5,13 @@ import random
|
||||
import time
|
||||
from typing import List, Tuple
|
||||
|
||||
from cacheflow import LLM, SamplingParams
|
||||
import torch
|
||||
from transformers import (AutoConfig, AutoTokenizer, AutoModelForCausalLM,
|
||||
PreTrainedTokenizerBase)
|
||||
from tqdm import tqdm
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
|
||||
|
||||
def get_tokenizer(model_name: str) -> PreTrainedTokenizerBase:
|
||||
config = AutoConfig.from_pretrained(model_name)
|
||||
@@ -70,7 +71,7 @@ def sample_requests(
|
||||
return sampled_requests
|
||||
|
||||
|
||||
def run_cacheflow(
|
||||
def run_vllm(
|
||||
requests: List[Tuple[str, int, int]],
|
||||
model: str,
|
||||
tensor_parallel_size: int,
|
||||
@@ -172,8 +173,8 @@ def main(args: argparse.Namespace):
|
||||
tokenizer = get_tokenizer(args.model)
|
||||
requests = sample_requests(args.dataset, args.num_prompts, tokenizer)
|
||||
|
||||
if args.backend == "cacheflow":
|
||||
elapsed_time = run_cacheflow(
|
||||
if args.backend == "vllm":
|
||||
elapsed_time = run_vllm(
|
||||
requests, args.model, args.tensor_parallel_size, args.seed, args.n,
|
||||
args.use_beam_search)
|
||||
elif args.backend == "hf":
|
||||
@@ -192,8 +193,8 @@ def main(args: argparse.Namespace):
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Benchmark the throughput.")
|
||||
parser.add_argument("--backend", type=str, choices=["cacheflow", "hf"],
|
||||
default="cacheflow")
|
||||
parser.add_argument("--backend", type=str, choices=["vllm", "hf"],
|
||||
default="vllm")
|
||||
parser.add_argument("--dataset", type=str, required=True,
|
||||
help="Path to the dataset.")
|
||||
parser.add_argument("--model", type=str, default="facebook/opt-125m")
|
||||
@@ -207,7 +208,7 @@ if __name__ == "__main__":
|
||||
parser.add_argument("--hf-max-batch-size", type=int, default=None,
|
||||
help="Maximum batch size for HF backend.")
|
||||
args = parser.parse_args()
|
||||
if args.backend == "cacheflow":
|
||||
if args.backend == "vllm":
|
||||
if args.hf_max_batch_size is not None:
|
||||
raise ValueError("HF max batch size is only for HF backend.")
|
||||
elif args.backend == "hf":
|
||||
|
||||
Reference in New Issue
Block a user