[Core][5/N] Fully working chunked prefill e2e (#3884)

This commit is contained in:
SangBin Cho
2024-04-10 17:56:48 -07:00
committed by GitHub
parent 63e7176f26
commit 67b4221a61
26 changed files with 927 additions and 315 deletions
+2 -3
View File
@@ -386,9 +386,8 @@ class EngineArgs:
'prompt latency) before scheduling next prompt.')
parser.add_argument(
'--enable-chunked-prefill',
type=bool,
default=False,
help='If True, the prefill requests can be chunked based on the '
action='store_true',
help='If set, the prefill requests can be chunked based on the '
'max_num_batched_tokens')
parser.add_argument(