[Bugfix] Max-load throughput cliff when max_num_seqs is not a multiple of 8 (#57355)

Signed-off-by: Andy Lo <andy@mistral.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Andy Lo
2026-09-17 18:09:18 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent ff6b5808c4
commit 09c379ab61
2 changed files with 24 additions and 3 deletions
+19
View File
@@ -541,6 +541,25 @@ def _mock_config_for_cudagraph_sizes(
return config
@pytest.mark.parametrize("max_num_seqs", [100, 101])
def test_default_cudagraph_capture_sizes_cover_off_stride_max_num_seqs(
max_num_seqs: int,
) -> None:
compilation_config = CompilationConfig(
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
)
config = _mock_config_for_cudagraph_sizes(
max_num_seqs=max_num_seqs,
num_speculative_tokens=0,
max_num_batched_tokens=32768,
compilation_config=compilation_config,
)
VllmConfig._set_cudagraph_sizes(config)
assert max_num_seqs in compilation_config.cudagraph_capture_sizes
@pytest.mark.parametrize(
("max_num_seqs", "num_speculative_tokens", "widest_is_captured"),
[
+5 -3
View File
@@ -2223,9 +2223,9 @@ class VllmConfig:
max_cudagraph_capture_size = (
self.compilation_config.max_cudagraph_capture_size
)
# Decode sizes to cover, in tokens. Populated only when a request
# is more than one token wide and only when the default is computed
# here, so an explicit capture range is left exactly as configured.
# Decode sizes to cover, in tokens. Populated only when the default
# is computed here, so an explicit capture range is left exactly as
# configured.
uniform_decode_sizes: list[int] = []
if max_cudagraph_capture_size is None:
from vllm.platforms import current_platform
@@ -2316,6 +2316,8 @@ class VllmConfig:
if n * query_len <= max_cudagraph_capture_size
}
)
elif max_num_seqs <= max_cudagraph_capture_size:
uniform_decode_sizes = [max_num_seqs]
max_num_tokens = self.scheduler_config.max_num_batched_tokens
max_cudagraph_capture_size = min(max_num_tokens, max_cudagraph_capture_size)