[Model Runner V2] support auto resolve cudagraph mode/sizes based on attn backend (#32936)
Signed-off-by: zhuhaoran <zhuhaoran.zhr@alibaba-inc.com>
This commit is contained in:
@@ -6307,7 +6307,7 @@ class GPUModelRunner(
|
||||
cudagraph_mode.
|
||||
"""
|
||||
min_cg_support = AttentionCGSupport.ALWAYS
|
||||
min_cg_backend_name = None
|
||||
min_cg_attn_backend = None
|
||||
|
||||
for attn_backend_set, kv_cache_group in zip(
|
||||
attention_backends, kv_cache_groups
|
||||
@@ -6320,152 +6320,18 @@ class GPUModelRunner(
|
||||
)
|
||||
if cg_support.value < min_cg_support.value:
|
||||
min_cg_support = cg_support
|
||||
min_cg_backend_name = attn_backend.__name__
|
||||
# Flexible resolve the cudagraph mode
|
||||
cudagraph_mode = self.compilation_config.cudagraph_mode
|
||||
assert cudagraph_mode is not None
|
||||
# check cudagraph for mixed batch is supported
|
||||
if (
|
||||
cudagraph_mode.mixed_mode() == CUDAGraphMode.FULL
|
||||
and min_cg_support != AttentionCGSupport.ALWAYS
|
||||
):
|
||||
msg = (
|
||||
f"CUDAGraphMode.{cudagraph_mode.name} is not supported "
|
||||
f"with {min_cg_backend_name} backend (support: "
|
||||
f"{min_cg_support})"
|
||||
)
|
||||
if min_cg_support == AttentionCGSupport.NEVER:
|
||||
# if not supported any full cudagraphs, just raise it.
|
||||
msg += (
|
||||
"; please try cudagraph_mode=PIECEWISE, and "
|
||||
"make sure compilation mode is VLLM_COMPILE"
|
||||
)
|
||||
raise ValueError(msg)
|
||||
|
||||
# attempt to resolve the full cudagraph related mode
|
||||
if self.compilation_config.splitting_ops_contain_attention():
|
||||
msg += "; setting cudagraph_mode=FULL_AND_PIECEWISE"
|
||||
cudagraph_mode = self.compilation_config.cudagraph_mode = (
|
||||
CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
)
|
||||
else:
|
||||
msg += "; setting cudagraph_mode=FULL_DECODE_ONLY"
|
||||
cudagraph_mode = self.compilation_config.cudagraph_mode = (
|
||||
CUDAGraphMode.FULL_DECODE_ONLY
|
||||
)
|
||||
logger.warning(msg)
|
||||
|
||||
# check that if we are doing decode full-cudagraphs it is supported
|
||||
if (
|
||||
cudagraph_mode.decode_mode() == CUDAGraphMode.FULL
|
||||
and min_cg_support == AttentionCGSupport.NEVER
|
||||
):
|
||||
msg = (
|
||||
f"CUDAGraphMode.{cudagraph_mode.name} is not supported "
|
||||
f"with {min_cg_backend_name} backend (support: "
|
||||
f"{min_cg_support})"
|
||||
)
|
||||
if self.compilation_config.mode == CompilationMode.VLLM_COMPILE and (
|
||||
self.compilation_config.splitting_ops_contain_attention()
|
||||
or self.compilation_config.use_inductor_graph_partition
|
||||
):
|
||||
msg += (
|
||||
"; setting cudagraph_mode=PIECEWISE because "
|
||||
"attention is compiled piecewise"
|
||||
)
|
||||
cudagraph_mode = self.compilation_config.cudagraph_mode = (
|
||||
CUDAGraphMode.PIECEWISE
|
||||
)
|
||||
else:
|
||||
msg += (
|
||||
"; setting cudagraph_mode=NONE because "
|
||||
"attention is not compiled piecewise"
|
||||
)
|
||||
cudagraph_mode = self.compilation_config.cudagraph_mode = (
|
||||
CUDAGraphMode.NONE
|
||||
)
|
||||
logger.warning(msg)
|
||||
|
||||
# check that if we are doing spec-decode + decode full-cudagraphs it is
|
||||
# supported
|
||||
if (
|
||||
cudagraph_mode.decode_mode() == CUDAGraphMode.FULL
|
||||
and self.uniform_decode_query_len > 1
|
||||
and min_cg_support.value < AttentionCGSupport.UNIFORM_BATCH.value
|
||||
):
|
||||
msg = (
|
||||
f"CUDAGraphMode.{cudagraph_mode.name} is not supported"
|
||||
f" with spec-decode for attention backend "
|
||||
f"{min_cg_backend_name} (support: {min_cg_support})"
|
||||
)
|
||||
if self.compilation_config.splitting_ops_contain_attention():
|
||||
msg += "; setting cudagraph_mode=PIECEWISE"
|
||||
cudagraph_mode = self.compilation_config.cudagraph_mode = (
|
||||
CUDAGraphMode.PIECEWISE
|
||||
)
|
||||
else:
|
||||
msg += "; setting cudagraph_mode=NONE"
|
||||
cudagraph_mode = self.compilation_config.cudagraph_mode = (
|
||||
CUDAGraphMode.NONE
|
||||
)
|
||||
logger.warning(msg)
|
||||
|
||||
# double check that we can support full cudagraph if they are requested
|
||||
# even after automatic downgrades
|
||||
if (
|
||||
cudagraph_mode.has_full_cudagraphs()
|
||||
and min_cg_support == AttentionCGSupport.NEVER
|
||||
):
|
||||
raise ValueError(
|
||||
f"CUDAGraphMode.{cudagraph_mode.name} is not "
|
||||
f"supported with {min_cg_backend_name} backend ("
|
||||
f"support:{min_cg_support}) "
|
||||
"; please try cudagraph_mode=PIECEWISE, "
|
||||
"and make sure compilation mode is VLLM_COMPILE"
|
||||
)
|
||||
|
||||
# if we have dedicated decode cudagraphs, and spec-decode is enabled,
|
||||
# we need to adjust the cudagraph sizes to be a multiple of the uniform
|
||||
# decode query length to avoid: https://github.com/vllm-project/vllm/issues/28207
|
||||
# temp-fix: https://github.com/vllm-project/vllm/issues/28207#issuecomment-3504004536
|
||||
# Will be removed in the near future when we have separate cudagraph capture
|
||||
# sizes for decode and mixed prefill-decode.
|
||||
if (
|
||||
cudagraph_mode.decode_mode() == CUDAGraphMode.FULL
|
||||
and cudagraph_mode.separate_routine()
|
||||
and self.uniform_decode_query_len > 1
|
||||
):
|
||||
self.compilation_config.adjust_cudagraph_sizes_for_spec_decode(
|
||||
self.uniform_decode_query_len, self.parallel_config.tensor_parallel_size
|
||||
)
|
||||
|
||||
# For Mamba models with FULL decode cudagraphs, each decode
|
||||
# sequence needs one Mamba cache block. The decode cudagraph
|
||||
# dispatcher already caps batch sizes at max_num_seqs, so we just
|
||||
# need to verify that enough blocks exist. Raising here instead
|
||||
# of silently capping cudagraph_capture_sizes avoids unintended
|
||||
# restrictions on PIECEWISE (prefill) cudagraphs.
|
||||
# See: https://github.com/vllm-project/vllm/issues/34094
|
||||
if cudagraph_mode.has_full_cudagraphs() and not is_profiling:
|
||||
has_mamba = any(
|
||||
isinstance(g.kv_cache_spec, MambaSpec) for g in kv_cache_groups
|
||||
)
|
||||
if has_mamba and self.kv_cache_config is not None:
|
||||
num_blocks = self.kv_cache_config.num_blocks
|
||||
if self.max_num_reqs > num_blocks:
|
||||
raise ValueError(
|
||||
f"max_num_seqs ({self.max_num_reqs}) exceeds "
|
||||
f"available Mamba cache blocks ({num_blocks}). "
|
||||
f"Each decode sequence requires one Mamba cache "
|
||||
f"block, so CUDA graph capture cannot proceed. "
|
||||
f"Please lower max_num_seqs to at most "
|
||||
f"{num_blocks} or increase "
|
||||
f"gpu_memory_utilization."
|
||||
)
|
||||
|
||||
min_cg_attn_backend = attn_backend.__name__
|
||||
cudagraph_mode = self.compilation_config.resolve_cudagraph_mode_and_sizes(
|
||||
min_cg_support,
|
||||
min_cg_attn_backend,
|
||||
self.uniform_decode_query_len,
|
||||
self.parallel_config.tensor_parallel_size,
|
||||
self.kv_cache_config,
|
||||
self.max_num_reqs,
|
||||
is_profiling=is_profiling,
|
||||
)
|
||||
# Trigger cudagraph dispatching keys initialization after
|
||||
# resolved cudagraph mode.
|
||||
self.compilation_config.cudagraph_mode = cudagraph_mode
|
||||
self.cudagraph_dispatcher.initialize_cudagraph_keys(
|
||||
cudagraph_mode, self.uniform_decode_query_len
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user