[V1] Prefix caching (take 2) (#9972)
Signed-off-by: Cody Yu <hao.yu.cody@gmail.com>
This commit is contained in:
@@ -65,6 +65,7 @@ class LLMEngine:
|
||||
elif usage_context == UsageContext.OPENAI_API_SERVER:
|
||||
scheduler_config.max_num_seqs = 1024
|
||||
scheduler_config.max_num_batched_tokens = 2048
|
||||
cache_config.enable_prefix_caching = True
|
||||
|
||||
logger.info(
|
||||
"Initializing an LLM engine (v%s) with config: "
|
||||
|
||||
Reference in New Issue
Block a user