[BUGFIX] Do not return ignored sentences twice in async llm engine (#2258)

This commit is contained in:
Zhuohan Li
2023-12-26 13:41:09 +08:00
committed by GitHub
parent face83c7ec
commit e0ff920001
2 changed files with 7 additions and 22 deletions
+4 -6
View File
@@ -183,20 +183,18 @@ class _AsyncLLMEngine(LLMEngine):
and updates the scheduler with the model outputs. Finally, it decodes
the sequences and returns the newly generated results.
"""
seq_group_metadata_list, scheduler_outputs, ignored = self._schedule()
if scheduler_outputs.is_empty():
return ignored
seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
# Execute the model.
output = await self._run_workers_async(
output = (await self._run_workers_async(
"execute_model",
seq_group_metadata_list=seq_group_metadata_list,
blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in,
blocks_to_swap_out=scheduler_outputs.blocks_to_swap_out,
blocks_to_copy=scheduler_outputs.blocks_to_copy,
)
)) if not scheduler_outputs.is_empty() else []
return self._process_model_outputs(output, scheduler_outputs) + ignored
return self._process_model_outputs(output, scheduler_outputs)
async def _run_workers_async(
self,