Add Automatic Prefix Caching (#2762)
Co-authored-by: ElizaWszola <eliza@neuralmagic.com> Co-authored-by: Michael Goin <michael@neuralmagic.com>
This commit is contained in:
co-authored by
ElizaWszola
Michael Goin
parent
baee28c46c
commit
ce4f5a29fb
@@ -37,20 +37,13 @@ for output in outputs:
|
||||
|
||||
print("-" * 80)
|
||||
|
||||
# -1 since the last token can change when concatenating prompts.
|
||||
prefix_pos = len(llm.llm_engine.tokenizer.encode(prefix)) - 1
|
||||
|
||||
# The llm.generate call will batch all prompts and send the batch at once if resources allow.
|
||||
# The prefix will only be cached after the first batch is processed, so we need to call generate once
|
||||
# to calculate the prefix and cache it.
|
||||
outputs = llm.generate(generating_prompts[0],
|
||||
sampling_params,
|
||||
prefix_pos=[prefix_pos])
|
||||
outputs = llm.generate(generating_prompts[0], sampling_params)
|
||||
|
||||
# Subsequent batches can leverage the cached prefix
|
||||
outputs = llm.generate(generating_prompts,
|
||||
sampling_params,
|
||||
prefix_pos=[prefix_pos] * len(generating_prompts))
|
||||
outputs = llm.generate(generating_prompts, sampling_params)
|
||||
|
||||
# Print the outputs. You should see the same outputs as before
|
||||
for output in outputs:
|
||||
|
||||
Reference in New Issue
Block a user