[ModelRunnerV2] Avoid pipeline parallel bubbles (#42187)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -67,9 +67,18 @@ class InputBatch:
|
||||
seq_lens_cpu_upper_bound: torch.Tensor
|
||||
# [num_reqs]
|
||||
dcp_local_seq_lens: torch.Tensor | None
|
||||
# [num_reqs] CPU bool array.
|
||||
# [num_reqs]
|
||||
num_computed_tokens_np: np.ndarray
|
||||
# [num_reqs]
|
||||
prefill_len_np: np.ndarray
|
||||
# [num_reqs]
|
||||
num_computed_prefill_tokens_np: np.ndarray
|
||||
# [num_reqs] CPU bool array == (num_computed_prefill_tokens_np < prefill_len_np).
|
||||
is_prefilling_np: np.ndarray
|
||||
|
||||
# [num_reqs] only populated when pipeline parallelism is enabled.
|
||||
max_seq_len_np: np.ndarray | None
|
||||
|
||||
# [num_tokens_after_padding]
|
||||
input_ids: torch.Tensor
|
||||
# [num_tokens_after_padding]
|
||||
@@ -148,7 +157,11 @@ class InputBatch:
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||
dcp_local_seq_lens=None,
|
||||
num_computed_tokens_np=np.zeros(num_reqs, dtype=np.int32),
|
||||
prefill_len_np=np.zeros(num_reqs, dtype=np.int32),
|
||||
num_computed_prefill_tokens_np=np.zeros(num_reqs, dtype=np.int32),
|
||||
is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_),
|
||||
max_seq_len_np=None,
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
logits_indices=logits_indices,
|
||||
@@ -438,6 +451,9 @@ def _post_update_kernel(
|
||||
):
|
||||
req_id = tl.program_id(0)
|
||||
req_state_idx = tl.load(idx_mapping_ptr + req_id)
|
||||
if req_state_idx < 0:
|
||||
# Filter rows with negative index entries.
|
||||
return
|
||||
|
||||
total_len = tl.load(total_len_ptr + req_state_idx)
|
||||
num_sampled = tl.load(num_sampled_ptr + req_id)
|
||||
@@ -464,18 +480,22 @@ def _post_update_kernel(
|
||||
count = tl.load(token_ptr)
|
||||
tl.store(token_ptr, count + 1)
|
||||
|
||||
query_start = tl.load(query_start_loc_ptr + req_id)
|
||||
query_end = tl.load(query_start_loc_ptr + req_id + 1)
|
||||
query_len = query_end - query_start
|
||||
if query_start_loc_ptr is None:
|
||||
query_len = 0
|
||||
else:
|
||||
query_start = tl.load(query_start_loc_ptr + req_id)
|
||||
query_end = tl.load(query_start_loc_ptr + req_id + 1)
|
||||
query_len = query_end - query_start
|
||||
num_rejected = tl.load(num_rejected_ptr + req_id)
|
||||
|
||||
num_computed = tl.load(num_computed_tokens_ptr + req_state_idx)
|
||||
num_computed += query_len - num_rejected
|
||||
tl.store(num_computed_tokens_ptr + req_state_idx, num_computed)
|
||||
computed_delta = query_len - num_rejected
|
||||
if computed_delta != 0:
|
||||
num_computed = tl.load(num_computed_tokens_ptr + req_state_idx)
|
||||
tl.store(num_computed_tokens_ptr + req_state_idx, num_computed + computed_delta)
|
||||
|
||||
|
||||
def post_update(
|
||||
# [num_reqs]
|
||||
# [num_reqs] batch_idx -> req_state_idx; negative index means skip.
|
||||
idx_mapping: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
num_computed_tokens: torch.Tensor,
|
||||
@@ -490,7 +510,7 @@ def post_update(
|
||||
# [num_reqs]
|
||||
num_rejected: torch.Tensor,
|
||||
# [num_reqs + 1]
|
||||
query_start_loc: torch.Tensor,
|
||||
query_start_loc: torch.Tensor | None,
|
||||
# [max_num_reqs, max_model_len]
|
||||
all_token_ids: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
@@ -516,7 +536,7 @@ def post_update(
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _post_update_pool_kernel(
|
||||
def _post_update_num_computed_tokens_kernel(
|
||||
idx_mapping_ptr,
|
||||
num_computed_tokens_ptr,
|
||||
query_start_loc_ptr,
|
||||
@@ -531,7 +551,7 @@ def _post_update_pool_kernel(
|
||||
tl.store(num_computed_tokens_ptr + req_state_idx, num_computed + query_len)
|
||||
|
||||
|
||||
def post_update_pool(
|
||||
def post_update_num_computed_tokens(
|
||||
# [num_reqs]
|
||||
idx_mapping: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
@@ -540,7 +560,7 @@ def post_update_pool(
|
||||
query_start_loc: torch.Tensor,
|
||||
) -> None:
|
||||
num_reqs = idx_mapping.shape[0]
|
||||
_post_update_pool_kernel[(num_reqs,)](
|
||||
_post_update_num_computed_tokens_kernel[(num_reqs,)](
|
||||
idx_mapping,
|
||||
num_computed_tokens,
|
||||
query_start_loc,
|
||||
|
||||
Reference in New Issue
Block a user