[ModelRunnerV2] Avoid pipeline parallel bubbles (#42187)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-06-02 14:02:01 -07:00
committed by GitHub
parent 557781131a
commit e15f20258b
19 changed files with 615 additions and 144 deletions
+32 -12
View File
@@ -67,9 +67,18 @@ class InputBatch:
seq_lens_cpu_upper_bound: torch.Tensor
# [num_reqs]
dcp_local_seq_lens: torch.Tensor | None
# [num_reqs] CPU bool array.
# [num_reqs]
num_computed_tokens_np: np.ndarray
# [num_reqs]
prefill_len_np: np.ndarray
# [num_reqs]
num_computed_prefill_tokens_np: np.ndarray
# [num_reqs] CPU bool array == (num_computed_prefill_tokens_np < prefill_len_np).
is_prefilling_np: np.ndarray
# [num_reqs] only populated when pipeline parallelism is enabled.
max_seq_len_np: np.ndarray | None
# [num_tokens_after_padding]
input_ids: torch.Tensor
# [num_tokens_after_padding]
@@ -148,7 +157,11 @@ class InputBatch:
seq_lens=seq_lens,
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
dcp_local_seq_lens=None,
num_computed_tokens_np=np.zeros(num_reqs, dtype=np.int32),
prefill_len_np=np.zeros(num_reqs, dtype=np.int32),
num_computed_prefill_tokens_np=np.zeros(num_reqs, dtype=np.int32),
is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_),
max_seq_len_np=None,
input_ids=input_ids,
positions=positions,
logits_indices=logits_indices,
@@ -438,6 +451,9 @@ def _post_update_kernel(
):
req_id = tl.program_id(0)
req_state_idx = tl.load(idx_mapping_ptr + req_id)
if req_state_idx < 0:
# Filter rows with negative index entries.
return
total_len = tl.load(total_len_ptr + req_state_idx)
num_sampled = tl.load(num_sampled_ptr + req_id)
@@ -464,18 +480,22 @@ def _post_update_kernel(
count = tl.load(token_ptr)
tl.store(token_ptr, count + 1)
query_start = tl.load(query_start_loc_ptr + req_id)
query_end = tl.load(query_start_loc_ptr + req_id + 1)
query_len = query_end - query_start
if query_start_loc_ptr is None:
query_len = 0
else:
query_start = tl.load(query_start_loc_ptr + req_id)
query_end = tl.load(query_start_loc_ptr + req_id + 1)
query_len = query_end - query_start
num_rejected = tl.load(num_rejected_ptr + req_id)
num_computed = tl.load(num_computed_tokens_ptr + req_state_idx)
num_computed += query_len - num_rejected
tl.store(num_computed_tokens_ptr + req_state_idx, num_computed)
computed_delta = query_len - num_rejected
if computed_delta != 0:
num_computed = tl.load(num_computed_tokens_ptr + req_state_idx)
tl.store(num_computed_tokens_ptr + req_state_idx, num_computed + computed_delta)
def post_update(
# [num_reqs]
# [num_reqs] batch_idx -> req_state_idx; negative index means skip.
idx_mapping: torch.Tensor,
# [max_num_reqs]
num_computed_tokens: torch.Tensor,
@@ -490,7 +510,7 @@ def post_update(
# [num_reqs]
num_rejected: torch.Tensor,
# [num_reqs + 1]
query_start_loc: torch.Tensor,
query_start_loc: torch.Tensor | None,
# [max_num_reqs, max_model_len]
all_token_ids: torch.Tensor,
# [max_num_reqs]
@@ -516,7 +536,7 @@ def post_update(
@triton.jit
def _post_update_pool_kernel(
def _post_update_num_computed_tokens_kernel(
idx_mapping_ptr,
num_computed_tokens_ptr,
query_start_loc_ptr,
@@ -531,7 +551,7 @@ def _post_update_pool_kernel(
tl.store(num_computed_tokens_ptr + req_state_idx, num_computed + query_len)
def post_update_pool(
def post_update_num_computed_tokens(
# [num_reqs]
idx_mapping: torch.Tensor,
# [max_num_reqs]
@@ -540,7 +560,7 @@ def post_update_pool(
query_start_loc: torch.Tensor,
) -> None:
num_reqs = idx_mapping.shape[0]
_post_update_pool_kernel[(num_reqs,)](
_post_update_num_computed_tokens_kernel[(num_reqs,)](
idx_mapping,
num_computed_tokens,
query_start_loc,