[Frontend][V1] Online serving performance improvements (#12287)
This commit is contained in:
+36
-16
@@ -2,9 +2,12 @@ import asyncio
|
||||
import os
|
||||
from typing import AsyncGenerator, List, Mapping, Optional, Type, Union
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.envs import VLLM_V1_OUTPUT_PROC_CHUNK_SIZE
|
||||
from vllm.inputs import INPUT_REGISTRY, InputRegistry, PromptType
|
||||
from vllm.inputs.preprocess import InputPreprocessor
|
||||
from vllm.logger import init_logger
|
||||
@@ -16,7 +19,7 @@ from vllm.sampling_params import SamplingParams
|
||||
from vllm.transformers_utils.tokenizer import AnyTokenizer
|
||||
from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
|
||||
from vllm.usage.usage_lib import UsageContext
|
||||
from vllm.utils import kill_process_tree
|
||||
from vllm.utils import cdiv, kill_process_tree
|
||||
from vllm.v1.engine.core_client import EngineCoreClient
|
||||
from vllm.v1.engine.output_processor import OutputProcessor
|
||||
from vllm.v1.engine.processor import Processor
|
||||
@@ -205,17 +208,15 @@ class AsyncLLM(EngineClient):
|
||||
|
||||
# The output_handler task pushes items into the queue.
|
||||
# This task pulls from the queue and yields to caller.
|
||||
while True:
|
||||
finished = False
|
||||
while not finished:
|
||||
# Note: drain queue without await if possible (avoids
|
||||
# task switching under load which helps performance).
|
||||
out = q.get_nowait() if q.qsize() > 0 else await q.get()
|
||||
out = q.get_nowait() if not q.empty() else await q.get()
|
||||
|
||||
# Note: both OutputProcessor and EngineCore handle their
|
||||
# own request cleanup based on finished.
|
||||
if out.finished:
|
||||
yield out
|
||||
break
|
||||
|
||||
finished = out.finished
|
||||
yield out
|
||||
|
||||
# If the request is disconnected by the client, the
|
||||
@@ -233,22 +234,41 @@ class AsyncLLM(EngineClient):
|
||||
# 1) Pull EngineCoreOutputs from the EngineCore.
|
||||
outputs = await self.engine_core.get_output_async()
|
||||
|
||||
# 2) Process EngineCoreOutputs.
|
||||
processed_outputs = self.output_processor.process_outputs(
|
||||
outputs.outputs)
|
||||
# NOTE: RequestOutputs are pushed to their queues.
|
||||
assert len(processed_outputs.request_outputs) == 0
|
||||
# Split outputs into chunks of at most
|
||||
# VLLM_V1_OUTPUT_PROC_CHUNK_SIZE, so that we don't block the
|
||||
# event loop for too long.
|
||||
num_outputs = len(outputs.outputs)
|
||||
if num_outputs <= VLLM_V1_OUTPUT_PROC_CHUNK_SIZE:
|
||||
slices = (outputs.outputs, )
|
||||
else:
|
||||
slices = np.array_split(
|
||||
outputs.outputs,
|
||||
cdiv(num_outputs, VLLM_V1_OUTPUT_PROC_CHUNK_SIZE))
|
||||
|
||||
# 3) Abort any reqs that finished due to stop strings.
|
||||
await self.engine_core.abort_requests_async(
|
||||
processed_outputs.reqs_to_abort)
|
||||
iteration_stats = None
|
||||
for i, outputs_slice in enumerate(slices):
|
||||
# 2) Process EngineCoreOutputs.
|
||||
processed_outputs = self.output_processor.process_outputs(
|
||||
outputs_slice, iteration_stats)
|
||||
# NOTE: RequestOutputs are pushed to their queues.
|
||||
assert not processed_outputs.request_outputs
|
||||
iteration_stats = processed_outputs.iteration_stats
|
||||
|
||||
# Allow other asyncio tasks to run between chunks
|
||||
if i + 1 < len(slices):
|
||||
await asyncio.sleep(0)
|
||||
|
||||
# 3) Abort any reqs that finished due to stop strings.
|
||||
await self.engine_core.abort_requests_async(
|
||||
processed_outputs.reqs_to_abort)
|
||||
|
||||
# 4) Logging.
|
||||
# TODO(rob): make into a coroutine and launch it in
|
||||
# background thread once we add Prometheus.
|
||||
assert iteration_stats is not None
|
||||
self._log_stats(
|
||||
scheduler_stats=outputs.scheduler_stats,
|
||||
iteration_stats=processed_outputs.iteration_stats,
|
||||
iteration_stats=iteration_stats,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import asyncio
|
||||
import os
|
||||
import signal
|
||||
import weakref
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import List, Type
|
||||
from typing import List, Optional, Type
|
||||
|
||||
import msgspec
|
||||
import zmq
|
||||
@@ -255,10 +256,24 @@ class AsyncMPClient(MPClient):
|
||||
log_stats=True,
|
||||
)
|
||||
|
||||
async def get_output_async(self) -> EngineCoreOutputs:
|
||||
self.outputs_queue: Optional[asyncio.Queue[bytes]] = None
|
||||
self.queue_task: Optional[asyncio.Task] = None
|
||||
|
||||
frames = await self.output_socket.recv_multipart(copy=False)
|
||||
return self.decoder.decode(frames[0].buffer)
|
||||
async def get_output_async(self) -> EngineCoreOutputs:
|
||||
if self.outputs_queue is None:
|
||||
# Perform IO in separate task to parallelize as much as possible
|
||||
self.outputs_queue = asyncio.Queue()
|
||||
|
||||
async def process_outputs_socket():
|
||||
assert self.outputs_queue is not None
|
||||
while True:
|
||||
(frame, ) = await self.output_socket.recv_multipart(
|
||||
copy=False)
|
||||
self.outputs_queue.put_nowait(frame.buffer)
|
||||
|
||||
self.queue_task = asyncio.create_task(process_outputs_socket())
|
||||
|
||||
return self.decoder.decode(await self.outputs_queue.get())
|
||||
|
||||
async def _send_input(self, request_type: EngineCoreRequestType,
|
||||
request: EngineCoreRequestUnion) -> None:
|
||||
|
||||
@@ -101,6 +101,7 @@ class OutputProcessor:
|
||||
def process_outputs(
|
||||
self,
|
||||
engine_core_outputs: List[EngineCoreOutput],
|
||||
iteration_stats: Optional[IterationStats] = None,
|
||||
) -> OutputProcessorOutput:
|
||||
"""
|
||||
Process the EngineCoreOutputs:
|
||||
@@ -133,7 +134,8 @@ class OutputProcessor:
|
||||
|
||||
request_outputs: List[RequestOutput] = []
|
||||
reqs_to_abort: List[str] = []
|
||||
iteration_stats = IterationStats(self.log_stats)
|
||||
if not iteration_stats:
|
||||
iteration_stats = IterationStats(self.log_stats)
|
||||
for engine_core_output in engine_core_outputs:
|
||||
req_id = engine_core_output.request_id
|
||||
req_state = self.request_states.get(req_id)
|
||||
@@ -175,8 +177,8 @@ class OutputProcessor:
|
||||
iteration_stats=iteration_stats,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _make_request_output(
|
||||
self,
|
||||
request_state: RequestState,
|
||||
detokenizer_output: Optional[DetokenizerOutput],
|
||||
) -> Optional[RequestOutput]:
|
||||
|
||||
Reference in New Issue
Block a user