[Core] Async_output_proc: Add virtual engine support (towards pipeline parallel) (#7911)
This commit is contained in:
@@ -91,7 +91,7 @@ class ModelInputForGPU(ModelRunnerInputBase):
|
||||
request_ids_to_seq_ids: Optional[Dict[str, List[int]]] = None
|
||||
finished_requests_ids: Optional[List[str]] = None
|
||||
virtual_engine: int = 0
|
||||
output_proc_callback_fn: Optional[Callable] = None
|
||||
async_callback: Optional[Callable] = None
|
||||
|
||||
def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
|
||||
tensor_dict = {
|
||||
@@ -1457,8 +1457,8 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
|
||||
if not self.is_driver_worker:
|
||||
return []
|
||||
|
||||
if model_input.output_proc_callback_fn is not None:
|
||||
model_input.output_proc_callback_fn(is_async=True)
|
||||
if model_input.async_callback is not None:
|
||||
model_input.async_callback()
|
||||
|
||||
# Sample the next token.
|
||||
output: SamplerOutput = self.model.sample(
|
||||
|
||||
Reference in New Issue
Block a user