[Model Runner V2] Enable piecewise CUDA graphs for pipeline parallelism (#35162)
Signed-off-by: Zhanqiu Hu <zh338@cornell.edu> Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
@@ -11,11 +11,16 @@ from tqdm import tqdm
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.distributed.parallel_state import graph_capture, is_global_first_rank
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_pp_group,
|
||||
graph_capture,
|
||||
is_global_first_rank,
|
||||
)
|
||||
from vllm.forward_context import BatchDescriptor, set_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.offloader.base import get_offloader
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
@@ -87,7 +92,15 @@ class CudaGraphManager:
|
||||
assert self.compilation_config is not None
|
||||
self.cudagraph_mode = cudagraph_mode
|
||||
self.decode_query_len = decode_query_len
|
||||
|
||||
self.dp_size = vllm_config.parallel_config.data_parallel_size
|
||||
self.pp_size = vllm_config.parallel_config.pipeline_parallel_size
|
||||
if self.pp_size > 1:
|
||||
self.is_first_pp_rank = get_pp_group().is_first_rank
|
||||
self.is_last_pp_rank = get_pp_group().is_last_rank
|
||||
else:
|
||||
self.is_first_pp_rank = True
|
||||
self.is_last_pp_rank = True
|
||||
|
||||
self.graphs: dict[BatchExecutionDescriptor, torch.cuda.CUDAGraph] = {}
|
||||
self.pool = current_platform.get_global_graph_pool() if cudagraph_mode else None
|
||||
@@ -267,12 +280,14 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
self.hidden_states: torch.Tensor | None = None
|
||||
self.aux_hidden_states: list[torch.Tensor] = []
|
||||
self.use_aux_hidden_state_outputs = False
|
||||
self.intermediate_tensors: IntermediateTensors | None = None
|
||||
|
||||
def capture(
|
||||
self,
|
||||
model: nn.Module,
|
||||
model_state: ModelState,
|
||||
input_buffers: InputBuffers,
|
||||
intermediate_tensors: IntermediateTensors | None,
|
||||
block_tables: BlockTables,
|
||||
attn_groups: list[list[AttentionGroup]],
|
||||
kv_cache_config: KVCacheConfig,
|
||||
@@ -293,6 +308,19 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
if self.dp_size > 1
|
||||
else None
|
||||
)
|
||||
|
||||
model_inputs = {
|
||||
"input_ids": input_buffers.input_ids[:num_tokens],
|
||||
"positions": input_buffers.positions[:num_tokens],
|
||||
**model_state.prepare_dummy_inputs(num_reqs, num_tokens),
|
||||
}
|
||||
if not self.is_first_pp_rank:
|
||||
# Update for non-first PP ranks.
|
||||
model_inputs["input_ids"] = None
|
||||
model_inputs["inputs_embeds"] = None
|
||||
assert intermediate_tensors is not None
|
||||
model_inputs["intermediate_tensors"] = intermediate_tensors[:num_tokens]
|
||||
|
||||
attn_metadata, slot_mappings = prepare_inputs_to_capture(
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
@@ -318,21 +346,15 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
slot_mapping=slot_mappings,
|
||||
batch_descriptor=batch_descriptor,
|
||||
):
|
||||
model_inputs = {
|
||||
"input_ids": input_buffers.input_ids[:num_tokens],
|
||||
"positions": input_buffers.positions[:num_tokens],
|
||||
# TODO: Pass intermediate_tensors for PP CUDA graph
|
||||
# support (https://github.com/vllm-project/vllm/pull/35162).
|
||||
"intermediate_tensors": None,
|
||||
**model_state.prepare_dummy_inputs(num_reqs, num_tokens),
|
||||
}
|
||||
model_output = model(**model_inputs)
|
||||
|
||||
if cg_mode == CUDAGraphMode.PIECEWISE:
|
||||
# PW CUDA graph internally handles the model outputs.
|
||||
# No need to keep track of the hidden states.
|
||||
return None
|
||||
if cg_mode == CUDAGraphMode.PIECEWISE:
|
||||
# PW CUDA graph internally handles the model outputs.
|
||||
# No need to keep track of the hidden states.
|
||||
return None
|
||||
|
||||
if self.is_last_pp_rank:
|
||||
# Last PP rank (common case).
|
||||
if self.use_aux_hidden_state_outputs:
|
||||
hidden_states, aux_hidden_states = model_output
|
||||
else:
|
||||
@@ -340,13 +362,26 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
aux_hidden_states = []
|
||||
if self.hidden_states is None:
|
||||
self.hidden_states = torch.empty_like(hidden_states)
|
||||
self.hidden_states[:num_tokens] = hidden_states
|
||||
if self.use_aux_hidden_state_outputs and not self.aux_hidden_states:
|
||||
self.aux_hidden_states = [
|
||||
torch.empty_like(x) for x in aux_hidden_states
|
||||
]
|
||||
self.hidden_states[:num_tokens] = hidden_states
|
||||
for i, aux in enumerate(aux_hidden_states):
|
||||
self.aux_hidden_states[i][:num_tokens] = aux
|
||||
else:
|
||||
# Non-last PP rank.
|
||||
intermediate_tensors = model_output
|
||||
assert isinstance(intermediate_tensors, IntermediateTensors)
|
||||
if self.intermediate_tensors is None:
|
||||
self.intermediate_tensors = IntermediateTensors(
|
||||
{
|
||||
k: torch.empty_like(v)
|
||||
for k, v in intermediate_tensors.tensors.items()
|
||||
}
|
||||
)
|
||||
for k, v in intermediate_tensors.tensors.items():
|
||||
self.intermediate_tensors[k][:num_tokens] = v
|
||||
|
||||
return forward_fn
|
||||
|
||||
@@ -354,9 +389,13 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
|
||||
def run_fullgraph(
|
||||
self, desc: BatchExecutionDescriptor
|
||||
) -> torch.Tensor | tuple[torch.Tensor, list[torch.Tensor]]:
|
||||
) -> torch.Tensor | tuple[torch.Tensor, list[torch.Tensor]] | IntermediateTensors:
|
||||
"""Replay a captured FULL cudagraph and return hidden states."""
|
||||
super().run_fullgraph(desc)
|
||||
if not self.is_last_pp_rank:
|
||||
assert self.intermediate_tensors is not None
|
||||
return self.intermediate_tensors[: desc.num_tokens]
|
||||
|
||||
assert self.hidden_states is not None
|
||||
hidden_states = self.hidden_states[: desc.num_tokens]
|
||||
if not self.use_aux_hidden_state_outputs:
|
||||
|
||||
Reference in New Issue
Block a user