Rename servers to engines (#152)

This commit is contained in:
Zhuohan Li
2023-06-17 17:25:21 +08:00
committed by GitHub
parent bab8f3dd0d
commit e5464ee484
15 changed files with 165 additions and 174 deletions
+1 -1
View File
@@ -14,7 +14,7 @@ def main(args: argparse.Namespace):
# Process all the requests in a single batch if possible.
# NOTE(woosuk): If the request cannot be processed in a single batch,
# the server will automatically process the request in multiple batches.
# the engine will automatically process the request in multiple batches.
llm = LLM(
model=args.model,
tensor_parallel_size=args.tensor_parallel_size,
+1 -1
View File
@@ -2,7 +2,7 @@
On the server side, run one of the following commands:
(CacheFlow backend)
python -m cacheflow.entrypoints.simple_fastapi_frontend \
python -m cacheflow.entrypoints.api_server \
--disable-log-requests --model <your_model>
(TGI backend)
+2 -2
View File
@@ -84,7 +84,7 @@ def run_cacheflow(
seed=seed,
)
# Add the requests to the server.
# Add the requests to the engine.
for prompt, _, output_len in requests:
sampling_params = SamplingParams(
n=n,
@@ -103,7 +103,7 @@ def run_cacheflow(
start = time.time()
# FIXME(woosuk): Do use internal method.
llm._run_server(use_tqdm=True)
llm._run_engine(use_tqdm=True)
end = time.time()
return end - start