[Core] Consolidate prompt arguments to LLM engines (#4328)
Co-authored-by: Roger Wang <ywang@roblox.com>
This commit is contained in:
co-authored by
Roger Wang
parent
290f4ada2b
commit
5ae5ed1e60
@@ -1,18 +1,20 @@
|
||||
from typing import List
|
||||
from typing import Sequence as GenericSequence
|
||||
from typing import Union
|
||||
|
||||
from vllm.sequence import SamplerOutput, SequenceGroupOutput
|
||||
from vllm.sequence import PoolerOutput, SamplerOutput, SequenceGroupOutput
|
||||
|
||||
|
||||
def create_output_by_sequence_group(
|
||||
sampler_outputs: List[SamplerOutput],
|
||||
outputs: GenericSequence[Union[SamplerOutput, PoolerOutput]],
|
||||
num_seq_groups: int) -> List[List[SequenceGroupOutput]]:
|
||||
"""Helper method which transforms a 2d list organized by
|
||||
[step][sequence group] into [sequence group][step].
|
||||
"""
|
||||
output_by_sequence_group: List[List[SamplerOutput]] = [
|
||||
output_by_sequence_group: List[List[SequenceGroupOutput]] = [
|
||||
[] for _ in range(num_seq_groups)
|
||||
]
|
||||
for step in sampler_outputs:
|
||||
for step in outputs:
|
||||
for i, sequence_group_output in enumerate(step):
|
||||
output_by_sequence_group[i].append(sequence_group_output)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user