[MISC] cudagraph_capture_sizes related improvements (#26016)

Signed-off-by: fhl <2410591650@qq.com>
Signed-off-by: fhl2000 <63384265+fhl2000@users.noreply.github.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
fhl2000
2025-10-24 05:11:05 -07:00
committed by GitHub
co-authored by Luka Govedič Harry Mellor
parent 435be10db9
commit 284cc92275
14 changed files with 303 additions and 110 deletions
+62 -5
View File
@@ -364,7 +364,13 @@ class EngineArgs:
kv_cache_dtype: CacheDType = CacheConfig.cache_dtype
seed: int | None = ModelConfig.seed
max_model_len: int | None = ModelConfig.max_model_len
cuda_graph_sizes: list[int] = get_field(SchedulerConfig, "cuda_graph_sizes")
cuda_graph_sizes: list[int] | None = CompilationConfig.cudagraph_capture_sizes
cudagraph_capture_sizes: list[int] | None = (
CompilationConfig.cudagraph_capture_sizes
)
max_cudagraph_capture_size: int | None = get_field(
CompilationConfig, "max_cudagraph_capture_size"
)
# Note: Specifying a custom executor backend by passing a class
# is intended for expert use only. The API may change without
# notice.
@@ -1007,9 +1013,6 @@ class EngineArgs:
"--max-long-partial-prefills",
**scheduler_kwargs["max_long_partial_prefills"],
)
scheduler_group.add_argument(
"--cuda-graph-sizes", **scheduler_kwargs["cuda_graph_sizes"]
)
scheduler_group.add_argument(
"--long-prefill-token-threshold",
**scheduler_kwargs["long_prefill_token_threshold"],
@@ -1039,6 +1042,29 @@ class EngineArgs:
"--async-scheduling", **scheduler_kwargs["async_scheduling"]
)
# Compilation arguments
compilation_kwargs = get_kwargs(CompilationConfig)
compilation_group = parser.add_argument_group(
title="CompilationConfig",
description=CompilationConfig.__doc__,
)
compilation_group.add_argument(
"--cudagraph-capture-sizes", **compilation_kwargs["cudagraph_capture_sizes"]
)
compilation_kwargs["cudagraph_capture_sizes"]["help"] = (
"--cuda-graph-sizes is deprecated and will be removed in v0.13.0 or v1.0.0,"
" whichever is soonest. Please use --cudagraph-capture-sizes instead."
)
compilation_group.add_argument(
"--cuda-graph-sizes",
**compilation_kwargs["cudagraph_capture_sizes"],
deprecated=True,
)
compilation_group.add_argument(
"--max-cudagraph-capture-size",
**compilation_kwargs["max_cudagraph_capture_size"],
)
# vLLM arguments
vllm_kwargs = get_kwargs(VllmConfig)
vllm_group = parser.add_argument_group(
@@ -1548,7 +1574,6 @@ class EngineArgs:
max_num_batched_tokens=self.max_num_batched_tokens,
max_num_seqs=self.max_num_seqs,
max_model_len=model_config.max_model_len,
cuda_graph_sizes=self.cuda_graph_sizes,
num_lookahead_slots=num_lookahead_slots,
enable_chunked_prefill=self.enable_chunked_prefill,
disable_chunked_mm_input=self.disable_chunked_mm_input,
@@ -1616,6 +1641,38 @@ class EngineArgs:
collect_detailed_traces=self.collect_detailed_traces,
)
# Compilation config overrides
if self.cuda_graph_sizes is not None:
logger.warning(
"--cuda-graph-sizes is deprecated and will be removed in v0.13.0 or "
"v1.0.0, whichever is soonest. Please use --cudagraph-capture-sizes "
"instead."
)
if self.compilation_config.cudagraph_capture_sizes is not None:
raise ValueError(
"cuda_graph_sizes and compilation_config."
"cudagraph_capture_sizes are mutually exclusive"
)
self.compilation_config.cudagraph_capture_sizes = self.cuda_graph_sizes
if self.cudagraph_capture_sizes is not None:
if self.compilation_config.cudagraph_capture_sizes is not None:
raise ValueError(
"cudagraph_capture_sizes and compilation_config."
"cudagraph_capture_sizes are mutually exclusive"
)
self.compilation_config.cudagraph_capture_sizes = (
self.cudagraph_capture_sizes
)
if self.max_cudagraph_capture_size is not None:
if self.compilation_config.max_cudagraph_capture_size is not None:
raise ValueError(
"max_cudagraph_capture_size and compilation_config."
"max_cudagraph_capture_size are mutually exclusive"
)
self.compilation_config.max_cudagraph_capture_size = (
self.max_cudagraph_capture_size
)
config = VllmConfig(
model_config=model_config,
cache_config=cache_config,