[Core][Bugfix]Refactor block manager for better testability (#3492)

This commit is contained in:
Cade Daniel
2024-03-27 23:59:28 -07:00
committed by GitHub
parent 8267b06c30
commit 14ccd94c89
30 changed files with 3285 additions and 77 deletions
-6
View File
@@ -553,12 +553,6 @@ class LLMEngine:
# Update the scheduled sequence groups with the model outputs.
scheduled_seq_groups = scheduler_outputs.scheduled_seq_groups
# If prefix caching is enabled, mark all blocks in the sequence groups
# as completed so that future requests don't attempt to recompute them
if self.cache_config.enable_prefix_caching:
for seq_group in scheduled_seq_groups:
self.scheduler.mark_blocks_as_computed(seq_group)
for seq_group, outputs in zip(scheduled_seq_groups, output):
self._process_sequence_group_outputs(seq_group, outputs)