[Perf] Change default CUDAGraphMode from PIECEWISE to FULL_AND_PIECEWISE (#25444)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -2947,8 +2947,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
# TODO(luka) better system for describing dummy batches
|
||||
seq_lens = [1] * num_decode_tokens + [num_prefill_tokens + 1]
|
||||
else:
|
||||
# Make sure max_model_len is used at the graph capture time.
|
||||
seq_lens = self.max_model_len
|
||||
seq_lens = max_query_len
|
||||
self.seq_lens.np[:num_reqs] = seq_lens
|
||||
self.seq_lens.np[num_reqs:] = 0
|
||||
self.seq_lens.copy_to_gpu()
|
||||
@@ -3541,6 +3540,26 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
CUDAGraphMode.FULL_DECODE_ONLY
|
||||
logger.warning(msg)
|
||||
|
||||
# check that if we are doing decode full-cudagraphs it is supported
|
||||
if (cudagraph_mode.decode_mode() == CUDAGraphMode.FULL
|
||||
and min_cg_support == AttentionCGSupport.NEVER):
|
||||
msg = (f"CUDAGraphMode.{cudagraph_mode.name} is not supported "
|
||||
f"with {min_cg_builder_name} backend (support: "
|
||||
f"{min_cg_support})")
|
||||
if (self.compilation_config.level == CompilationLevel.PIECEWISE and
|
||||
(self.compilation_config.splitting_ops_contain_attention()
|
||||
or self.compilation_config.use_inductor_graph_partition)):
|
||||
msg += "; setting cudagraph_mode=PIECEWISE because "\
|
||||
"attention is compiled piecewise"
|
||||
cudagraph_mode = self.compilation_config.cudagraph_mode = \
|
||||
CUDAGraphMode.PIECEWISE
|
||||
else:
|
||||
msg += "; setting cudagraph_mode=NONE because "\
|
||||
"attention is not compiled piecewise"
|
||||
cudagraph_mode = self.compilation_config.cudagraph_mode = \
|
||||
CUDAGraphMode.NONE
|
||||
logger.warning(msg)
|
||||
|
||||
# check that if we are doing spec-decode + decode full-cudagraphs it is
|
||||
# supported
|
||||
if (cudagraph_mode.decode_mode() == CUDAGraphMode.FULL
|
||||
|
||||
Reference in New Issue
Block a user