Rename servers to engines (#152)
This commit is contained in:
@@ -14,7 +14,7 @@ def main(args: argparse.Namespace):
|
||||
|
||||
# Process all the requests in a single batch if possible.
|
||||
# NOTE(woosuk): If the request cannot be processed in a single batch,
|
||||
# the server will automatically process the request in multiple batches.
|
||||
# the engine will automatically process the request in multiple batches.
|
||||
llm = LLM(
|
||||
model=args.model,
|
||||
tensor_parallel_size=args.tensor_parallel_size,
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
On the server side, run one of the following commands:
|
||||
(CacheFlow backend)
|
||||
python -m cacheflow.entrypoints.simple_fastapi_frontend \
|
||||
python -m cacheflow.entrypoints.api_server \
|
||||
--disable-log-requests --model <your_model>
|
||||
|
||||
(TGI backend)
|
||||
|
||||
@@ -84,7 +84,7 @@ def run_cacheflow(
|
||||
seed=seed,
|
||||
)
|
||||
|
||||
# Add the requests to the server.
|
||||
# Add the requests to the engine.
|
||||
for prompt, _, output_len in requests:
|
||||
sampling_params = SamplingParams(
|
||||
n=n,
|
||||
@@ -103,7 +103,7 @@ def run_cacheflow(
|
||||
|
||||
start = time.time()
|
||||
# FIXME(woosuk): Do use internal method.
|
||||
llm._run_server(use_tqdm=True)
|
||||
llm._run_engine(use_tqdm=True)
|
||||
end = time.time()
|
||||
return end - start
|
||||
|
||||
|
||||
Reference in New Issue
Block a user