[V1] Prefix caching (take 2) (#9972)

Signed-off-by: Cody Yu <hao.yu.cody@gmail.com>
This commit is contained in:
Cody Yu
2024-11-07 17:34:44 -08:00
committed by GitHub
parent 42b4f46b71
commit 201fc07730
6 changed files with 770 additions and 65 deletions
+1
View File
@@ -65,6 +65,7 @@ class LLMEngine:
elif usage_context == UsageContext.OPENAI_API_SERVER:
scheduler_config.max_num_seqs = 1024
scheduler_config.max_num_batched_tokens = 2048
cache_config.enable_prefix_caching = True
logger.info(
"Initializing an LLM engine (v%s) with config: "