[Doc] Chunked Prefill Documentation (#4580)
This commit is contained in:
+3
-2
@@ -607,8 +607,9 @@ class SchedulerConfig:
|
||||
self.max_num_batched_tokens = max_num_batched_tokens
|
||||
else:
|
||||
if enable_chunked_prefill:
|
||||
# For chunked prefill, choose the well-tuned batch size.
|
||||
self.max_num_batched_tokens = 768
|
||||
# It is the values that have the best balance between ITL
|
||||
# and TTFT on A100. Note it is not optimized for throughput.
|
||||
self.max_num_batched_tokens = 512
|
||||
else:
|
||||
# If max_model_len is too short, use 2048 as the default value
|
||||
# for higher throughput.
|
||||
|
||||
Reference in New Issue
Block a user