[ModelRunnerV2] Avoid pipeline parallel bubbles (#42187)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -61,7 +61,10 @@ from vllm.v1.worker.gpu.attn_utils import (
|
||||
init_kv_cache,
|
||||
)
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu
|
||||
from vllm.v1.worker.gpu.buffer_utils import (
|
||||
async_copy_to_gpu,
|
||||
set_default_max_concurrency,
|
||||
)
|
||||
from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
BatchExecutionDescriptor,
|
||||
@@ -77,7 +80,7 @@ from vllm.v1.worker.gpu.input_batch import (
|
||||
expand_idx_mapping,
|
||||
get_num_sampled_and_rejected,
|
||||
post_update,
|
||||
post_update_pool,
|
||||
post_update_num_computed_tokens,
|
||||
prepare_pos_seq_lens,
|
||||
prepare_prefill_inputs,
|
||||
)
|
||||
@@ -90,7 +93,7 @@ from vllm.v1.worker.gpu.lora_utils import LoraState
|
||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||
from vllm.v1.worker.gpu.model_states import init_model_state
|
||||
from vllm.v1.worker.gpu.pool.pooling_runner import PoolingRunner
|
||||
from vllm.v1.worker.gpu.pp_utils import pp_broadcast, pp_receive
|
||||
from vllm.v1.worker.gpu.pp_utils import PPHandler
|
||||
from vllm.v1.worker.gpu.sample.output import SamplerOutput
|
||||
from vllm.v1.worker.gpu.sample.prompt_logprob import PromptLogprobsWorker
|
||||
from vllm.v1.worker.gpu.sample.sampler import Sampler
|
||||
@@ -149,6 +152,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.is_first_pp_rank = get_pp_group().is_first_rank
|
||||
self.is_last_pp_rank = get_pp_group().is_last_rank
|
||||
|
||||
# Size the UVA buffer pools to the max number of concurrent in-flight
|
||||
# steps. Must run before any pooled buffer is constructed
|
||||
set_default_max_concurrency(vllm_config.max_concurrent_batches)
|
||||
|
||||
# PP broadcast/recv helper. Runs the collective on a side stream.
|
||||
self.pp_handler: PPHandler | None = None
|
||||
|
||||
# Persistent buffer for intermediate tensors (non-first PP ranks).
|
||||
self.intermediate_tensors: IntermediateTensors | None = None
|
||||
|
||||
@@ -210,6 +220,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
if self.use_pp:
|
||||
self.pp_handler = PPHandler(
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
num_speculative_steps=self.num_speculative_steps,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
self.sampler: Sampler | None = None
|
||||
self.rejection_sampler: RejectionSampler | None = None
|
||||
self.prompt_logprobs_worker: PromptLogprobsWorker | None = None
|
||||
@@ -688,8 +705,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
return cuda_graph_size
|
||||
|
||||
def _remove_request(self, req_id: str) -> bool:
|
||||
if not self.req_states.remove_request(req_id):
|
||||
req_idx = self.req_states.remove_request(req_id)
|
||||
if req_idx is None:
|
||||
return False
|
||||
if self.pp_handler is not None:
|
||||
self.pp_handler.on_req_idx_freed(req_idx)
|
||||
if self.encoder_cache is not None:
|
||||
self.encoder_cache.remove_request(req_id)
|
||||
if self.prompt_logprobs_worker is not None:
|
||||
@@ -710,6 +730,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
for mm_hash in scheduler_output.free_encoder_mm_hashes:
|
||||
self.encoder_cache.free_encoder_cache(mm_hash)
|
||||
|
||||
def update_pp_decode_requests(self):
|
||||
# For non-last PP ranks, update decode requests with sampler output from
|
||||
# the prior step in which they were scheduled (pp_size steps ago).
|
||||
if self.pp_handler is not None:
|
||||
outputs = self.pp_handler.get_prev_sampled_outputs()
|
||||
if outputs is not None:
|
||||
self.postprocess_sampled(**outputs)
|
||||
|
||||
def add_requests(self, scheduler_output: SchedulerOutput) -> None:
|
||||
for new_req_data in scheduler_output.scheduled_new_reqs:
|
||||
assert new_req_data.prompt_token_ids is not None
|
||||
@@ -722,11 +750,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self._remove_request(req_id)
|
||||
|
||||
prompt_len = len(new_req_data.prompt_token_ids)
|
||||
sampling_params = new_req_data.sampling_params
|
||||
self.req_states.add_request(
|
||||
req_id=req_id,
|
||||
prompt_len=prompt_len,
|
||||
all_token_ids=new_req_data.prefill_token_ids,
|
||||
num_computed_tokens=new_req_data.num_computed_tokens,
|
||||
max_tokens=sampling_params.max_tokens if sampling_params else 1, # type: ignore[arg-type]
|
||||
)
|
||||
req_index = self.req_states.req_id_to_index[req_id]
|
||||
|
||||
@@ -769,7 +799,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
req_index, req_new_block_ids, overwrite=False
|
||||
)
|
||||
|
||||
# Update num_computed_prefill_tokens.
|
||||
# Update CPU num_computed_prefill_tokens.
|
||||
np.minimum(
|
||||
self.req_states.num_computed_tokens_np,
|
||||
self.req_states.prefill_len.np,
|
||||
@@ -848,7 +878,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
async_copy_to_gpu(query_start_loc_np, out=self.input_buffers.query_start_loc)
|
||||
query_start_loc_np = query_start_loc_np[: num_reqs_padded + 1]
|
||||
query_start_loc = self.input_buffers.query_start_loc[: num_reqs_padded + 1]
|
||||
is_prefilling_np = self.req_states.is_prefilling(idx_mapping_np)
|
||||
prefill_len_np = self.req_states.prefill_len.np[idx_mapping_np]
|
||||
computed_prefill_tokens_np = self.req_states.num_computed_prefill_tokens
|
||||
num_computed_prefill_tokens_np = computed_prefill_tokens_np[idx_mapping_np]
|
||||
is_prefilling_np = num_computed_prefill_tokens_np < prefill_len_np
|
||||
|
||||
# Get prefill tokens if any.
|
||||
if np.any(is_prefilling_np):
|
||||
@@ -900,13 +933,19 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
)
|
||||
|
||||
# CPU upper bound on seq_lens; padded entries left at zero.
|
||||
num_computed_tokens_np = self.req_states.num_computed_tokens_np[idx_mapping_np]
|
||||
seq_lens_cpu_upper_bound_np = np.zeros(num_reqs_padded, dtype=np.int32)
|
||||
np.add(
|
||||
self.req_states.num_computed_tokens_np[idx_mapping_np],
|
||||
num_computed_tokens_np,
|
||||
num_scheduled_tokens,
|
||||
out=seq_lens_cpu_upper_bound_np[:num_reqs],
|
||||
)
|
||||
seq_lens_cpu_upper_bound = torch.from_numpy(seq_lens_cpu_upper_bound_np)
|
||||
|
||||
max_seq_len_np = None
|
||||
if self.use_pp:
|
||||
# max_seq_len is only consumed by the PP `compute_need_sampled_mask`
|
||||
max_seq_len_np = self.req_states.max_seq_len[idx_mapping_np]
|
||||
return InputBatch(
|
||||
req_ids=req_ids,
|
||||
num_reqs=num_reqs,
|
||||
@@ -925,7 +964,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||
dcp_local_seq_lens=dcp_local_seq_lens,
|
||||
num_computed_tokens_np=num_computed_tokens_np,
|
||||
prefill_len_np=prefill_len_np,
|
||||
num_computed_prefill_tokens_np=num_computed_prefill_tokens_np,
|
||||
is_prefilling_np=is_prefilling_np,
|
||||
max_seq_len_np=max_seq_len_np,
|
||||
input_ids=self.input_buffers.input_ids[:num_tokens_after_padding],
|
||||
positions=self.input_buffers.positions[:num_tokens_after_padding],
|
||||
logits_indices=logits_indices,
|
||||
@@ -1005,12 +1048,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
)
|
||||
return sampler_output, num_sampled, num_rejected
|
||||
|
||||
def postprocess(
|
||||
def postprocess_sampled(
|
||||
self,
|
||||
input_batch: InputBatch,
|
||||
idx_mapping: torch.Tensor, # May include -1 for masked entries
|
||||
sampled_tokens: torch.Tensor,
|
||||
num_sampled: torch.Tensor,
|
||||
num_rejected: torch.Tensor,
|
||||
query_start_loc: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
# Update the number of computed tokens.
|
||||
if self.is_last_pp_rank:
|
||||
@@ -1019,19 +1063,19 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
else:
|
||||
output_bin_counts = None
|
||||
post_update(
|
||||
input_batch.idx_mapping,
|
||||
idx_mapping,
|
||||
self.req_states.num_computed_tokens.gpu,
|
||||
self.req_states.last_sampled_tokens,
|
||||
output_bin_counts,
|
||||
sampled_tokens,
|
||||
num_sampled,
|
||||
num_rejected,
|
||||
input_batch.query_start_loc,
|
||||
query_start_loc,
|
||||
self.req_states.all_token_ids.gpu,
|
||||
self.req_states.total_len.gpu,
|
||||
)
|
||||
|
||||
self.model_state.postprocess_state(input_batch, num_sampled)
|
||||
self.model_state.postprocess_state(idx_mapping, num_sampled)
|
||||
|
||||
@torch.inference_mode()
|
||||
def execute_model(
|
||||
@@ -1044,6 +1088,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
) -> ModelRunnerOutput | IntermediateTensors | None:
|
||||
if not dummy_run:
|
||||
# Update the request states.
|
||||
self.update_pp_decode_requests()
|
||||
self.finish_requests(scheduler_output)
|
||||
self.free_states(scheduler_output)
|
||||
self.add_requests(scheduler_output)
|
||||
@@ -1162,9 +1207,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# NOTE(woosuk): We must call get_mm_embeddings even during dummy runs
|
||||
# to obtain inputs_embeds, because the compiled model expects this input.
|
||||
inputs_embeds = self.model_state.get_mm_embeddings(
|
||||
scheduler_output.scheduled_encoder_inputs,
|
||||
input_batch,
|
||||
self.req_states,
|
||||
scheduler_output.scheduled_encoder_inputs, input_batch
|
||||
)
|
||||
|
||||
model_inputs = {
|
||||
@@ -1184,12 +1227,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
assert intermediate_tensors is not None
|
||||
assert self.intermediate_tensors is not None
|
||||
n = input_batch.num_tokens_after_padding
|
||||
model_inputs["intermediate_tensors"] = IntermediateTensors(
|
||||
{
|
||||
k: v[:n].copy_(intermediate_tensors.tensors[k][:n])
|
||||
for k, v in self.intermediate_tensors.tensors.items()
|
||||
}
|
||||
)
|
||||
new_tensors = {
|
||||
k: v[:n]
|
||||
if dummy_run
|
||||
else v[:n].copy_(intermediate_tensors.tensors[k][:n])
|
||||
for k, v in self.intermediate_tensors.tensors.items()
|
||||
}
|
||||
model_inputs["intermediate_tensors"] = IntermediateTensors(new_tensors)
|
||||
del intermediate_tensors
|
||||
|
||||
# Run model.
|
||||
@@ -1281,10 +1325,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# Non-last PP rank: hidden_states is None because this rank produced
|
||||
# IntermediateTensors instead of final hidden states. Receive the
|
||||
# sampled tokens broadcast from the last rank and update local state.
|
||||
sampled, num_sampled, num_rejected = pp_receive(
|
||||
input_batch.num_reqs, max_sample_len=self.num_speculative_steps + 1
|
||||
)
|
||||
self.postprocess(input_batch, sampled, num_sampled, num_rejected)
|
||||
assert self.pp_handler is not None
|
||||
all_decode_next = self.pp_handler.receive(input_batch)
|
||||
# Optimistically update num_computed_tokens for entire batch here.
|
||||
# Will be adjusted for rejections if necessary in update_requests.
|
||||
self.postprocess_num_computed_tokens(input_batch)
|
||||
if not all_decode_next:
|
||||
# Might contain non-final prefill chunks, which will be scheduled
|
||||
# in the immediate next step (rather than in pp_size steps).
|
||||
self.model_state.postprocess_state(input_batch.idx_mapping, 0)
|
||||
|
||||
# Post-step KV connector related operations.
|
||||
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
|
||||
@@ -1295,9 +1344,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
hidden_states, input_batch, grammar_output
|
||||
)
|
||||
|
||||
if self.use_pp:
|
||||
if self.pp_handler is not None:
|
||||
# Broadcast to non-last PP ranks (handles spec decode multi-token).
|
||||
pp_broadcast(sampler_output.sampled_token_ids, num_sampled, num_rejected)
|
||||
self.pp_handler.broadcast(
|
||||
sampler_output.sampled_token_ids,
|
||||
num_sampled,
|
||||
num_rejected,
|
||||
input_batch,
|
||||
)
|
||||
|
||||
assert self.prompt_logprobs_worker is not None
|
||||
prompt_logprobs_dict = self.prompt_logprobs_worker.compute_prompt_logprobs(
|
||||
@@ -1307,8 +1361,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.req_states.all_token_ids.gpu,
|
||||
self.req_states.num_computed_tokens.gpu,
|
||||
self.req_states.prompt_len.np,
|
||||
self.req_states.prefill_len.np,
|
||||
self.req_states.num_computed_prefill_tokens,
|
||||
)
|
||||
|
||||
# Prepare the model runner output.
|
||||
@@ -1334,17 +1386,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# Get cached multimodal embeddings for draft forward.
|
||||
# NOTE: This is done here because postprocess updates
|
||||
# num_computed_prefill_tokens.
|
||||
prefill_lens = self.req_states.prefill_len.np[input_batch.idx_mapping_np]
|
||||
computed_prefill_lens = self.req_states.num_computed_prefill_tokens[
|
||||
input_batch.idx_mapping_np
|
||||
]
|
||||
mm_inputs = self.model_state.encoder_runner.gather_mm_embeddings(
|
||||
input_batch.req_ids,
|
||||
input_batch.num_tokens,
|
||||
input_batch.num_scheduled_tokens,
|
||||
input_batch.query_start_loc_np,
|
||||
prefill_lens,
|
||||
computed_prefill_lens + 1, # +1 to consider the skew in eagle
|
||||
input_batch.prefill_len_np,
|
||||
# +1 to consider the skew in eagle
|
||||
input_batch.num_computed_prefill_tokens_np + 1,
|
||||
)
|
||||
|
||||
# Postprocess results and update request states.
|
||||
@@ -1352,8 +1401,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# ensuring that `copy_event` is recorded before calling postprocess.
|
||||
# This sequencing may slightly reduce latency as async D2H copy does not
|
||||
# need to wait for the postprocess to finish.
|
||||
self.postprocess(
|
||||
input_batch, sampler_output.sampled_token_ids, num_sampled, num_rejected
|
||||
self.postprocess_sampled(
|
||||
input_batch.idx_mapping,
|
||||
sampler_output.sampled_token_ids,
|
||||
num_sampled,
|
||||
num_rejected,
|
||||
input_batch.query_start_loc,
|
||||
)
|
||||
|
||||
if self.speculator is not None:
|
||||
@@ -1410,7 +1463,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
|
||||
|
||||
if not self.is_last_pp_rank:
|
||||
self.postprocess_pool(input_batch)
|
||||
self.postprocess_num_computed_tokens(input_batch)
|
||||
return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
|
||||
|
||||
assert self.pooling_runner is not None
|
||||
@@ -1432,14 +1485,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
copy_stream=self.output_copy_stream,
|
||||
)
|
||||
|
||||
self.postprocess_pool(input_batch)
|
||||
self.postprocess_num_computed_tokens(input_batch)
|
||||
if self.use_async_scheduling:
|
||||
return async_output
|
||||
return async_output.get_output()
|
||||
|
||||
def postprocess_pool(self, input_batch: InputBatch) -> None:
|
||||
def postprocess_num_computed_tokens(self, input_batch: InputBatch) -> None:
|
||||
# Update the number of computed tokens.
|
||||
post_update_pool(
|
||||
post_update_num_computed_tokens(
|
||||
input_batch.idx_mapping,
|
||||
self.req_states.num_computed_tokens.gpu,
|
||||
input_batch.query_start_loc,
|
||||
|
||||
Reference in New Issue
Block a user