[ModelRunnerV2] Avoid pipeline parallel bubbles (#42187)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-06-02 14:02:01 -07:00
committed by GitHub
parent 557781131a
commit e15f20258b
19 changed files with 615 additions and 144 deletions
+94 -41
View File
@@ -61,7 +61,10 @@ from vllm.v1.worker.gpu.attn_utils import (
init_kv_cache,
)
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu
from vllm.v1.worker.gpu.buffer_utils import (
async_copy_to_gpu,
set_default_max_concurrency,
)
from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens
from vllm.v1.worker.gpu.cudagraph_utils import (
BatchExecutionDescriptor,
@@ -77,7 +80,7 @@ from vllm.v1.worker.gpu.input_batch import (
expand_idx_mapping,
get_num_sampled_and_rejected,
post_update,
post_update_pool,
post_update_num_computed_tokens,
prepare_pos_seq_lens,
prepare_prefill_inputs,
)
@@ -90,7 +93,7 @@ from vllm.v1.worker.gpu.lora_utils import LoraState
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
from vllm.v1.worker.gpu.model_states import init_model_state
from vllm.v1.worker.gpu.pool.pooling_runner import PoolingRunner
from vllm.v1.worker.gpu.pp_utils import pp_broadcast, pp_receive
from vllm.v1.worker.gpu.pp_utils import PPHandler
from vllm.v1.worker.gpu.sample.output import SamplerOutput
from vllm.v1.worker.gpu.sample.prompt_logprob import PromptLogprobsWorker
from vllm.v1.worker.gpu.sample.sampler import Sampler
@@ -149,6 +152,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.is_first_pp_rank = get_pp_group().is_first_rank
self.is_last_pp_rank = get_pp_group().is_last_rank
# Size the UVA buffer pools to the max number of concurrent in-flight
# steps. Must run before any pooled buffer is constructed
set_default_max_concurrency(vllm_config.max_concurrent_batches)
# PP broadcast/recv helper. Runs the collective on a side stream.
self.pp_handler: PPHandler | None = None
# Persistent buffer for intermediate tensors (non-first PP ranks).
self.intermediate_tensors: IntermediateTensors | None = None
@@ -210,6 +220,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
device=self.device,
)
if self.use_pp:
self.pp_handler = PPHandler(
max_num_reqs=self.max_num_reqs,
num_speculative_steps=self.num_speculative_steps,
device=self.device,
)
self.sampler: Sampler | None = None
self.rejection_sampler: RejectionSampler | None = None
self.prompt_logprobs_worker: PromptLogprobsWorker | None = None
@@ -688,8 +705,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
return cuda_graph_size
def _remove_request(self, req_id: str) -> bool:
if not self.req_states.remove_request(req_id):
req_idx = self.req_states.remove_request(req_id)
if req_idx is None:
return False
if self.pp_handler is not None:
self.pp_handler.on_req_idx_freed(req_idx)
if self.encoder_cache is not None:
self.encoder_cache.remove_request(req_id)
if self.prompt_logprobs_worker is not None:
@@ -710,6 +730,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
for mm_hash in scheduler_output.free_encoder_mm_hashes:
self.encoder_cache.free_encoder_cache(mm_hash)
def update_pp_decode_requests(self):
# For non-last PP ranks, update decode requests with sampler output from
# the prior step in which they were scheduled (pp_size steps ago).
if self.pp_handler is not None:
outputs = self.pp_handler.get_prev_sampled_outputs()
if outputs is not None:
self.postprocess_sampled(**outputs)
def add_requests(self, scheduler_output: SchedulerOutput) -> None:
for new_req_data in scheduler_output.scheduled_new_reqs:
assert new_req_data.prompt_token_ids is not None
@@ -722,11 +750,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self._remove_request(req_id)
prompt_len = len(new_req_data.prompt_token_ids)
sampling_params = new_req_data.sampling_params
self.req_states.add_request(
req_id=req_id,
prompt_len=prompt_len,
all_token_ids=new_req_data.prefill_token_ids,
num_computed_tokens=new_req_data.num_computed_tokens,
max_tokens=sampling_params.max_tokens if sampling_params else 1, # type: ignore[arg-type]
)
req_index = self.req_states.req_id_to_index[req_id]
@@ -769,7 +799,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
req_index, req_new_block_ids, overwrite=False
)
# Update num_computed_prefill_tokens.
# Update CPU num_computed_prefill_tokens.
np.minimum(
self.req_states.num_computed_tokens_np,
self.req_states.prefill_len.np,
@@ -848,7 +878,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
async_copy_to_gpu(query_start_loc_np, out=self.input_buffers.query_start_loc)
query_start_loc_np = query_start_loc_np[: num_reqs_padded + 1]
query_start_loc = self.input_buffers.query_start_loc[: num_reqs_padded + 1]
is_prefilling_np = self.req_states.is_prefilling(idx_mapping_np)
prefill_len_np = self.req_states.prefill_len.np[idx_mapping_np]
computed_prefill_tokens_np = self.req_states.num_computed_prefill_tokens
num_computed_prefill_tokens_np = computed_prefill_tokens_np[idx_mapping_np]
is_prefilling_np = num_computed_prefill_tokens_np < prefill_len_np
# Get prefill tokens if any.
if np.any(is_prefilling_np):
@@ -900,13 +933,19 @@ class GPUModelRunner(LoRAModelRunnerMixin):
)
# CPU upper bound on seq_lens; padded entries left at zero.
num_computed_tokens_np = self.req_states.num_computed_tokens_np[idx_mapping_np]
seq_lens_cpu_upper_bound_np = np.zeros(num_reqs_padded, dtype=np.int32)
np.add(
self.req_states.num_computed_tokens_np[idx_mapping_np],
num_computed_tokens_np,
num_scheduled_tokens,
out=seq_lens_cpu_upper_bound_np[:num_reqs],
)
seq_lens_cpu_upper_bound = torch.from_numpy(seq_lens_cpu_upper_bound_np)
max_seq_len_np = None
if self.use_pp:
# max_seq_len is only consumed by the PP `compute_need_sampled_mask`
max_seq_len_np = self.req_states.max_seq_len[idx_mapping_np]
return InputBatch(
req_ids=req_ids,
num_reqs=num_reqs,
@@ -925,7 +964,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
seq_lens=seq_lens,
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
dcp_local_seq_lens=dcp_local_seq_lens,
num_computed_tokens_np=num_computed_tokens_np,
prefill_len_np=prefill_len_np,
num_computed_prefill_tokens_np=num_computed_prefill_tokens_np,
is_prefilling_np=is_prefilling_np,
max_seq_len_np=max_seq_len_np,
input_ids=self.input_buffers.input_ids[:num_tokens_after_padding],
positions=self.input_buffers.positions[:num_tokens_after_padding],
logits_indices=logits_indices,
@@ -1005,12 +1048,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
)
return sampler_output, num_sampled, num_rejected
def postprocess(
def postprocess_sampled(
self,
input_batch: InputBatch,
idx_mapping: torch.Tensor, # May include -1 for masked entries
sampled_tokens: torch.Tensor,
num_sampled: torch.Tensor,
num_rejected: torch.Tensor,
query_start_loc: torch.Tensor | None = None,
) -> None:
# Update the number of computed tokens.
if self.is_last_pp_rank:
@@ -1019,19 +1063,19 @@ class GPUModelRunner(LoRAModelRunnerMixin):
else:
output_bin_counts = None
post_update(
input_batch.idx_mapping,
idx_mapping,
self.req_states.num_computed_tokens.gpu,
self.req_states.last_sampled_tokens,
output_bin_counts,
sampled_tokens,
num_sampled,
num_rejected,
input_batch.query_start_loc,
query_start_loc,
self.req_states.all_token_ids.gpu,
self.req_states.total_len.gpu,
)
self.model_state.postprocess_state(input_batch, num_sampled)
self.model_state.postprocess_state(idx_mapping, num_sampled)
@torch.inference_mode()
def execute_model(
@@ -1044,6 +1088,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
) -> ModelRunnerOutput | IntermediateTensors | None:
if not dummy_run:
# Update the request states.
self.update_pp_decode_requests()
self.finish_requests(scheduler_output)
self.free_states(scheduler_output)
self.add_requests(scheduler_output)
@@ -1162,9 +1207,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# NOTE(woosuk): We must call get_mm_embeddings even during dummy runs
# to obtain inputs_embeds, because the compiled model expects this input.
inputs_embeds = self.model_state.get_mm_embeddings(
scheduler_output.scheduled_encoder_inputs,
input_batch,
self.req_states,
scheduler_output.scheduled_encoder_inputs, input_batch
)
model_inputs = {
@@ -1184,12 +1227,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
assert intermediate_tensors is not None
assert self.intermediate_tensors is not None
n = input_batch.num_tokens_after_padding
model_inputs["intermediate_tensors"] = IntermediateTensors(
{
k: v[:n].copy_(intermediate_tensors.tensors[k][:n])
for k, v in self.intermediate_tensors.tensors.items()
}
)
new_tensors = {
k: v[:n]
if dummy_run
else v[:n].copy_(intermediate_tensors.tensors[k][:n])
for k, v in self.intermediate_tensors.tensors.items()
}
model_inputs["intermediate_tensors"] = IntermediateTensors(new_tensors)
del intermediate_tensors
# Run model.
@@ -1281,10 +1325,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# Non-last PP rank: hidden_states is None because this rank produced
# IntermediateTensors instead of final hidden states. Receive the
# sampled tokens broadcast from the last rank and update local state.
sampled, num_sampled, num_rejected = pp_receive(
input_batch.num_reqs, max_sample_len=self.num_speculative_steps + 1
)
self.postprocess(input_batch, sampled, num_sampled, num_rejected)
assert self.pp_handler is not None
all_decode_next = self.pp_handler.receive(input_batch)
# Optimistically update num_computed_tokens for entire batch here.
# Will be adjusted for rejections if necessary in update_requests.
self.postprocess_num_computed_tokens(input_batch)
if not all_decode_next:
# Might contain non-final prefill chunks, which will be scheduled
# in the immediate next step (rather than in pp_size steps).
self.model_state.postprocess_state(input_batch.idx_mapping, 0)
# Post-step KV connector related operations.
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
@@ -1295,9 +1344,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
hidden_states, input_batch, grammar_output
)
if self.use_pp:
if self.pp_handler is not None:
# Broadcast to non-last PP ranks (handles spec decode multi-token).
pp_broadcast(sampler_output.sampled_token_ids, num_sampled, num_rejected)
self.pp_handler.broadcast(
sampler_output.sampled_token_ids,
num_sampled,
num_rejected,
input_batch,
)
assert self.prompt_logprobs_worker is not None
prompt_logprobs_dict = self.prompt_logprobs_worker.compute_prompt_logprobs(
@@ -1307,8 +1361,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.req_states.all_token_ids.gpu,
self.req_states.num_computed_tokens.gpu,
self.req_states.prompt_len.np,
self.req_states.prefill_len.np,
self.req_states.num_computed_prefill_tokens,
)
# Prepare the model runner output.
@@ -1334,17 +1386,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# Get cached multimodal embeddings for draft forward.
# NOTE: This is done here because postprocess updates
# num_computed_prefill_tokens.
prefill_lens = self.req_states.prefill_len.np[input_batch.idx_mapping_np]
computed_prefill_lens = self.req_states.num_computed_prefill_tokens[
input_batch.idx_mapping_np
]
mm_inputs = self.model_state.encoder_runner.gather_mm_embeddings(
input_batch.req_ids,
input_batch.num_tokens,
input_batch.num_scheduled_tokens,
input_batch.query_start_loc_np,
prefill_lens,
computed_prefill_lens + 1, # +1 to consider the skew in eagle
input_batch.prefill_len_np,
# +1 to consider the skew in eagle
input_batch.num_computed_prefill_tokens_np + 1,
)
# Postprocess results and update request states.
@@ -1352,8 +1401,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# ensuring that `copy_event` is recorded before calling postprocess.
# This sequencing may slightly reduce latency as async D2H copy does not
# need to wait for the postprocess to finish.
self.postprocess(
input_batch, sampler_output.sampled_token_ids, num_sampled, num_rejected
self.postprocess_sampled(
input_batch.idx_mapping,
sampler_output.sampled_token_ids,
num_sampled,
num_rejected,
input_batch.query_start_loc,
)
if self.speculator is not None:
@@ -1410,7 +1463,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
if not self.is_last_pp_rank:
self.postprocess_pool(input_batch)
self.postprocess_num_computed_tokens(input_batch)
return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
assert self.pooling_runner is not None
@@ -1432,14 +1485,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
copy_stream=self.output_copy_stream,
)
self.postprocess_pool(input_batch)
self.postprocess_num_computed_tokens(input_batch)
if self.use_async_scheduling:
return async_output
return async_output.get_output()
def postprocess_pool(self, input_batch: InputBatch) -> None:
def postprocess_num_computed_tokens(self, input_batch: InputBatch) -> None:
# Update the number of computed tokens.
post_update_pool(
post_update_num_computed_tokens(
input_batch.idx_mapping,
self.req_states.num_computed_tokens.gpu,
input_batch.query_start_loc,