[V1] Refactor parallel sampling support (#13774)

Signed-off-by: Mark McLoughlin <markmc@redhat.com>
This commit is contained in:
Mark McLoughlin
2025-03-03 08:15:27 -08:00
committed by GitHub
parent f35f8e2242
commit 4167252eaf
5 changed files with 198 additions and 461 deletions
+44 -300
View File
@@ -1,69 +1,46 @@
# SPDX-License-Identifier: Apache-2.0
from collections.abc import AsyncGenerator, Mapping
from copy import copy
from typing import Optional, Protocol, Union
from typing import Callable, Optional, Union
from vllm.inputs import PromptType
from vllm.lora.request import LoRARequest
from vllm.outputs import RequestOutput
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.pooling_params import PoolingParams
from vllm.prompt_adapter.request import PromptAdapterRequest
from vllm.sampling_params import RequestOutputKind, SamplingParams
from vllm.utils import merge_async_iterators
from vllm.sampling_params import SamplingParams
class AsyncGenerateMethodType(Protocol):
def __call__(self,
prompt: PromptType,
sampling_params: SamplingParams,
request_id: str,
lora_request: Optional[LoRARequest] = None,
trace_headers: Optional[Mapping[str, str]] = None,
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
priority: int = 0) -> AsyncGenerator[RequestOutput, None]:
...
class SyncAddRequestMethodType(Protocol):
def __call__(self,
request_id: str,
prompt: PromptType,
params: Union[SamplingParams, PoolingParams],
arrival_time: Optional[float] = None,
lora_request: Optional[LoRARequest] = None,
trace_headers: Optional[Mapping[str, str]] = None,
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
priority: int = 0) -> None:
...
class ParallelSamplingRequest:
class ParentRequest:
"""Info, state & processing for parallel sampling request.
Store parent request ID and sampling params.
Facilitate generating child request sampling params.
Transform child request outputs into parent request
outputs.
When stream mode is disabled, then `self.request_output`
aggregates child request completions.
"""
request_id: str
sampling_params: SamplingParams
# To aggregate child completions when not streaming
output_aggregator: Optional[RequestOutput]
# To efficiently obtain child sampling params
cached_child_sampling_params: Optional[SamplingParams]
request_output: Optional[RequestOutput]
num_finished_completions: int
def __init__(self, request_id: str,
sampling_params: SamplingParams) -> None:
self.request_id = request_id
self.sampling_params = sampling_params
self.output_aggregator = None
self.cached_child_sampling_params = None
self.request_output = None
self.num_finished_completions = 0
@classmethod
def from_params(
cls,
request_id: str,
params: Union[SamplingParams, PoolingParams],
) -> Optional['ParentRequest']:
if not isinstance(params, SamplingParams) or params.n == 1:
return None
return cls(request_id, params)
def _get_child_sampling_params(
self,
@@ -96,47 +73,6 @@ class ParallelSamplingRequest:
child_sampling_params.seed = seed + index
return child_sampling_params
def _add_output(
self,
child_req_output: RequestOutput,
index: int,
) -> None:
"""Aggregate a parallel sampling child
request output.
Non-stream-mode (`output_kind == FINAL_ONLY`)
only. Inject correct parent request ID and
completion index.
Args:
child_req_output: a single request output
from a parallel sampling
child request.
index: index within `n` child
"""
self.num_finished_completions += 1
new_completion = child_req_output.outputs[0]
new_completion.index = index
if self.request_output is None:
# Save the first request output; reinstate
# original request ID; metrics are not
# supported for parallel sampling
child_req_output.request_id = self.request_id
child_req_output.metrics = None
self.request_output = child_req_output
else:
# Aggregate additional completion into request output
# Note: will be sorted by index later
self.request_output.outputs.append(new_completion)
def _get_final_request_output(self) -> RequestOutput:
"""Invariant: parent completion outputs sorted by index"""
assert self.request_output is not None
self.request_output.finished = True
self.request_output.outputs = sorted(self.request_output.outputs,
key=lambda x: x.index)
return self.request_output
def get_child_info(self, index: int) -> tuple[str, SamplingParams]:
"""Get child request ID and sampling params.
@@ -149,227 +85,35 @@ class ParallelSamplingRequest:
return (f"{index}_{self.request_id}",
self._get_child_sampling_params(index))
def process_output(
self,
child_req_output: RequestOutput,
index: int,
) -> Optional[RequestOutput]:
"""Filter, aggregate and transform parallel sampling
child request outputs.
If the parent request has `stream=false`
(`output_kind == FINAL_ONLY`), each child will also have
`output_kind == FINAL_ONLY`. All child request outputs
must be aggregated into a single request output, with
multiple completions. This request output is only returned
once `n` completions are aggregated.
If the parent request has `stream=true`
(`output_kind == DELTA`), each child will also have
`output_kind == DELTA`. All child request outputs
must be streamed directly to the caller.
Args:
child_req_output: a single child request output
index: index within `n` child requests
Returns:
`None`, unless a processed request output is ready to
send back to the caller.
"""
if self.output_kind != RequestOutputKind.FINAL_ONLY:
# stream=true: return child completions immediately
child_req_output.request_id = self.request_id
child_req_output.outputs[0].index = index
if child_req_output.finished:
# Parent request is complete if all child requests are
# complete.
self.num_finished_completions += 1
child_req_output.finished = (
self.num_finished_completions == self.n)
return child_req_output
# stream=false: aggregate child completions
self._add_output(child_req_output, index)
if self.num_finished_completions == self.n:
# Return aggregated request output after obtaining
# all completions
return self._get_final_request_output()
return None
async def wrap_child_async_generator(
self,
child_gen: AsyncGenerator[RequestOutput, None],
index: int,
) -> AsyncGenerator[RequestOutput, None]:
"""Output generator for a single parallel sampling
child request.
Each parallel sampling request triggers at
least two child requests. This generator
yields zero or more request outputs to
return to the caller, as they become
available.
Args:
child_gen: generator for child request
outputs.
index: index within the `n` child requests
Returns:
Yields zero or more request outputs to return
to the caller.
"""
async for out in child_gen:
if req_out := self.process_output(out, index):
yield req_out
@property
def n(self) -> int:
return self.sampling_params.n
@property
def output_kind(self) -> RequestOutputKind:
return self.sampling_params.output_kind
class SyncParallelSamplingManager:
def __init__(self):
# Parent req ID -> parent request manager
self.parent_reqs: dict[str, ParallelSamplingRequest] = {}
# Child req ID -> (child req index, parent req ID)
self.child_reqs: dict[str, tuple[int, str]] = {}
def _register_parent_request(self, req: ParallelSamplingRequest) -> None:
"""Register parallel sampling parent request."""
self.parent_reqs[req.request_id] = req
def _register_child_request(self, req_id: str, child_req_id: str,
index: int) -> None:
"""Register parallel sampling child request with parent.
Args:
req_id: parent request ID
child_req_id: child request ID
index: child request index within `n` child requests
"""
self.child_reqs[child_req_id] = (index, req_id)
def get_num_unfinished_requests(self, num_core_reqs: int) -> int:
"""Get the number of unfinished requests, correcting for parallel
sampling.
Args:
num_core_reqs: The number of unfinished requests in the engine core.
Returns:
Number of unfinished requests, where each parallel sampling req
counts as 1
"""
return num_core_reqs + len(self.parent_reqs) - len(self.child_reqs)
def add_request_parallel_sampling(
def make_request_output(
self,
add_request: SyncAddRequestMethodType,
request_id: str,
prompt: PromptType,
params: Union[SamplingParams, PoolingParams],
arrival_time: Optional[float] = None,
lora_request: Optional[LoRARequest] = None,
trace_headers: Optional[Mapping[str, str]] = None,
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
priority: int = 0,
) -> None:
"""Add sync parallel sampling request."""
req = ParallelSamplingRequest(request_id, params)
self._register_parent_request(req)
# Add n child requests with unique request IDs & random seeds and n=1
for idx in range(req.n):
child_req_id, child_params = req.get_child_info(idx)
self._register_child_request(request_id, child_req_id, idx)
add_request(request_id=child_req_id,
prompt=prompt,
params=child_params,
arrival_time=arrival_time,
lora_request=lora_request,
trace_headers=trace_headers,
prompt_adapter_request=prompt_adapter_request,
priority=priority) # type: ignore
final_only: bool,
completion_output: CompletionOutput,
new_request_output: Callable[[str], RequestOutput],
) -> Optional[RequestOutput]:
# Use an existing RequestOutput if we're aggregating
request_output = self.output_aggregator
def step(
self,
outputs: list[RequestOutput],
) -> list[RequestOutput]:
"""Build parallel sampling request outputs.
Extract child request outputs, aggregate them
into parent request output, and return parent
output when complete.
# Make new RequestOutput otherwise
if request_output is None:
request_output = new_request_output(self.request_id)
Do not modify `n=1` requests.
# Add a new completion
request_output.outputs.append(completion_output)
Args:
outputs: step request outputs. Mix of child request
outputs & `n=1` request outputs.
# If not streaming, aggregate until all child requests complete
if final_only and len(request_output.outputs) != self.n:
self.output_aggregator = request_output
return None
Return:
List of parallel sampling parent request outputs &
unmodified `n=1` request outputs passed-thru from input.
"""
if not (self.parent_reqs and outputs):
# Return unmodified
return outputs
agg_outputs = []
for output in outputs:
req_id = output.request_id
if child_req_entry := self.child_reqs.get(req_id, None):
# For each parallel sampling child request output:
(index, parent_req_id) = child_req_entry
req = self.parent_reqs[parent_req_id]
# Update parallel sampling request
if out := req.process_output(output, index):
# Return parent request output if complete;
# cleanup parent request bookkeeping.
agg_outputs.append(out)
del self.parent_reqs[parent_req_id]
# Cleanup child request bookkeeping.
del self.child_reqs[req_id]
else:
# Not a parallel sampling request output
agg_outputs.append(output)
return agg_outputs
# We're done aggregating
self.output_aggregator = None
async def generate_parallel_sampling_async(
generate: AsyncGenerateMethodType,
prompt: PromptType,
sampling_params: SamplingParams,
request_id: str,
lora_request: Optional[LoRARequest] = None,
trace_headers: Optional[Mapping[str, str]] = None,
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
priority: int = 0,
) -> AsyncGenerator[RequestOutput, None]:
"""Generate completions for async parallel sampling requests."""
parent_req = ParallelSamplingRequest(request_id, sampling_params)
# Aggregate generators for n child requests
gens: list[AsyncGenerator[RequestOutput, None]] = []
for idx in range(parent_req.n):
child_req_id, child_params = parent_req.get_child_info(idx)
child_gen = generate(
prompt=prompt,
sampling_params=child_params,
request_id=child_req_id,
lora_request=lora_request,
trace_headers=trace_headers,
prompt_adapter_request=prompt_adapter_request,
priority=priority,
) # type: ignore
gen = parent_req.wrap_child_async_generator(child_gen, idx)
gens.append(gen)
# Merge generators
async for _, out in merge_async_iterators(*gens):
yield out
# Parent completion output list must be sorted by index
request_output.outputs = sorted(request_output.outputs,
key=lambda x: x.index)
return request_output