[V1][Frontend] Coalesce bunched RequestOutputs (#12298)
Signed-off-by: Nick Hill <nhill@redhat.com> Co-authored-by: Robert Shaw <rshaw@neuralmagic.com>
This commit is contained in:
+21
-1
@@ -1,6 +1,6 @@
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Dict, Generic, List, Optional
|
||||
from typing import Dict, Generic, List, MutableSequence, Optional
|
||||
from typing import Sequence as GenericSequence
|
||||
from typing import Union
|
||||
|
||||
@@ -162,6 +162,26 @@ class RequestOutput:
|
||||
finished=finished,
|
||||
)
|
||||
|
||||
def add(self, next_output: "RequestOutput") -> None:
|
||||
"""Merge subsequent RequestOutput into this one"""
|
||||
|
||||
self.prompt = next_output.prompt
|
||||
self.prompt_token_ids = next_output.prompt_token_ids
|
||||
self.prompt_logprobs = next_output.prompt_logprobs
|
||||
self.finished |= next_output.finished
|
||||
|
||||
#TODO assuming n == 1 for now
|
||||
completion = self.outputs[0]
|
||||
next_completion = next_output.outputs[0]
|
||||
completion.text += next_completion.text
|
||||
if not isinstance(completion.token_ids, MutableSequence):
|
||||
completion.token_ids = list(completion.token_ids)
|
||||
completion.token_ids.extend(next_completion.token_ids)
|
||||
if next_completion.logprobs:
|
||||
assert completion.logprobs is not None
|
||||
completion.logprobs.extend(next_completion.logprobs)
|
||||
completion.cumulative_logprob = next_completion.cumulative_logprob
|
||||
|
||||
@classmethod
|
||||
def from_seq_group(
|
||||
cls, seq_group: SequenceGroup, use_cache: bool,
|
||||
|
||||
@@ -15,7 +15,7 @@ from vllm.lora.request import LoRARequest
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.prompt_adapter.request import PromptAdapterRequest
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
||||
from vllm.transformers_utils.tokenizer import AnyTokenizer
|
||||
from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
|
||||
from vllm.usage.usage_lib import UsageContext
|
||||
@@ -214,6 +214,14 @@ class AsyncLLM(EngineClient):
|
||||
# task switching under load which helps performance).
|
||||
out = q.get_nowait() if not q.empty() else await q.get()
|
||||
|
||||
# Coalesce any additional queued outputs
|
||||
while not q.empty():
|
||||
next_out = q.get_nowait()
|
||||
if sampling_params.output_kind == RequestOutputKind.DELTA:
|
||||
out.add(next_out)
|
||||
else:
|
||||
out = next_out
|
||||
|
||||
# Note: both OutputProcessor and EngineCore handle their
|
||||
# own request cleanup based on finished.
|
||||
finished = out.finished
|
||||
|
||||
Reference in New Issue
Block a user