[Core] Asynchronous Output Processor (#7049)
Co-authored-by: Alexander Matveev <alexm@neuralmagic.com>
This commit is contained in:
co-authored by
Alexander Matveev
parent
015e6cc252
commit
2eedede875
@@ -263,6 +263,12 @@ class LocalOrDistributedWorkerBase(WorkerBase):
|
||||
broadcast_data.update(kwargs)
|
||||
broadcast_tensor_dict(broadcast_data, src=0)
|
||||
|
||||
if execute_model_req.output_proc_callback_fn:
|
||||
model_input = dataclasses.replace( # type: ignore
|
||||
model_input,
|
||||
output_proc_callback_fn=execute_model_req.
|
||||
output_proc_callback_fn)
|
||||
|
||||
return model_input, worker_input, kwargs
|
||||
|
||||
def prepare_input(
|
||||
@@ -289,7 +295,7 @@ class LocalOrDistributedWorkerBase(WorkerBase):
|
||||
|
||||
def execute_model(
|
||||
self,
|
||||
execute_model_req: Optional[ExecuteModelRequest] = None
|
||||
execute_model_req: Optional[ExecuteModelRequest] = None,
|
||||
) -> Optional[List[SamplerOutput]]:
|
||||
"""Executes at least one model step on the given sequences, unless no
|
||||
sequences are provided."""
|
||||
|
||||
Reference in New Issue
Block a user