[Doc] Chunked Prefill Documentation (#4580)

This commit is contained in:
SangBin Cho
2024-05-04 00:18:00 -07:00
committed by GitHub
parent bc8ad68455
commit 36fb68f947
3 changed files with 42 additions and 2 deletions
+3 -2
View File
@@ -607,8 +607,9 @@ class SchedulerConfig:
self.max_num_batched_tokens = max_num_batched_tokens
else:
if enable_chunked_prefill:
# For chunked prefill, choose the well-tuned batch size.
self.max_num_batched_tokens = 768
# It is the values that have the best balance between ITL
# and TTFT on A100. Note it is not optimized for throughput.
self.max_num_batched_tokens = 512
else:
# If max_model_len is too short, use 2048 as the default value
# for higher throughput.