[Core] Asynchronous Output Processor (#7049)

Co-authored-by: Alexander Matveev <alexm@neuralmagic.com>
This commit is contained in:
Megha Agarwal
2024-08-26 20:53:20 -07:00
committed by GitHub
co-authored by Alexander Matveev
parent 015e6cc252
commit 2eedede875
21 changed files with 652 additions and 214 deletions
+7 -1
View File
@@ -263,6 +263,12 @@ class LocalOrDistributedWorkerBase(WorkerBase):
broadcast_data.update(kwargs)
broadcast_tensor_dict(broadcast_data, src=0)
if execute_model_req.output_proc_callback_fn:
model_input = dataclasses.replace( # type: ignore
model_input,
output_proc_callback_fn=execute_model_req.
output_proc_callback_fn)
return model_input, worker_input, kwargs
def prepare_input(
@@ -289,7 +295,7 @@ class LocalOrDistributedWorkerBase(WorkerBase):
def execute_model(
self,
execute_model_req: Optional[ExecuteModelRequest] = None
execute_model_req: Optional[ExecuteModelRequest] = None,
) -> Optional[List[SamplerOutput]]:
"""Executes at least one model step on the given sequences, unless no
sequences are provided."""