[Core] Multi-Step + Single Step Prefills via Chunked Prefill code path (#8378)

Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
This commit is contained in:
Varun Sundar Rabindranath
2024-09-27 13:32:07 -07:00
committed by GitHub
co-authored by Varun Sundar Rabindranath
parent c5d55356f9
commit c2ec430ab5
19 changed files with 513 additions and 108 deletions
+3 -2
View File
@@ -76,8 +76,9 @@ class MultiStepWorker(Worker):
frozen_model_input = model_input.frozen_model_input
assert frozen_model_input is not None
assert frozen_model_input.attn_metadata is not None
# clear the cached decode metadata so that it can be recomputed on
# the workers
# clear the cached metadata so that it can be recomputed on
# the workers.
frozen_model_input.attn_metadata._cached_prefill_metadata = None
frozen_model_input.attn_metadata._cached_decode_metadata = None
model_input.is_first_multi_step = is_first_multi_step