[Bugfix] Max-load throughput cliff when max_num_seqs is not a multiple of 8 (#57355)
Signed-off-by: Andy Lo <andy@mistral.ai> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
+5
-3
@@ -2223,9 +2223,9 @@ class VllmConfig:
|
||||
max_cudagraph_capture_size = (
|
||||
self.compilation_config.max_cudagraph_capture_size
|
||||
)
|
||||
# Decode sizes to cover, in tokens. Populated only when a request
|
||||
# is more than one token wide and only when the default is computed
|
||||
# here, so an explicit capture range is left exactly as configured.
|
||||
# Decode sizes to cover, in tokens. Populated only when the default
|
||||
# is computed here, so an explicit capture range is left exactly as
|
||||
# configured.
|
||||
uniform_decode_sizes: list[int] = []
|
||||
if max_cudagraph_capture_size is None:
|
||||
from vllm.platforms import current_platform
|
||||
@@ -2316,6 +2316,8 @@ class VllmConfig:
|
||||
if n * query_len <= max_cudagraph_capture_size
|
||||
}
|
||||
)
|
||||
elif max_num_seqs <= max_cudagraph_capture_size:
|
||||
uniform_decode_sizes = [max_num_seqs]
|
||||
max_num_tokens = self.scheduler_config.max_num_batched_tokens
|
||||
max_cudagraph_capture_size = min(max_num_tokens, max_cudagraph_capture_size)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user