[DSpark] Support pipeline-parallel targets in aggregated serving (#56956)
Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com> Co-authored-by: Kimi Code <noreply@moonshot.cn> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
co-authored by
Kimi Code
Yongye Zhu
parent
15ed1262e7
commit
157bcb7c48
@@ -256,6 +256,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
slot_mapping=slot_mappings,
|
||||
batch_descriptor=batch_descriptor,
|
||||
is_padding=self.input_buffers.is_padding[:num_tokens],
|
||||
):
|
||||
last_hidden_states = self.model(
|
||||
input_ids=self.input_buffers.input_ids[:num_tokens],
|
||||
@@ -487,6 +488,7 @@ def _prepare_dflash_inputs_kernel(
|
||||
# Outputs
|
||||
out_input_ids_ptr,
|
||||
out_query_positions_ptr,
|
||||
out_is_padding_ptr,
|
||||
out_query_start_loc_ptr,
|
||||
out_seq_lens_ptr,
|
||||
out_query_slot_mapping_ptr,
|
||||
@@ -617,6 +619,7 @@ def _prepare_dflash_inputs_kernel(
|
||||
)
|
||||
|
||||
tl.store(out_input_ids_ptr + query_idx, input_id, mask=is_query)
|
||||
tl.store(out_is_padding_ptr + query_idx, False, mask=is_query)
|
||||
clamped_query_pos = tl.minimum(query_pos, max_model_len - 1)
|
||||
tl.store(out_query_positions_ptr + query_idx, clamped_query_pos, mask=is_query)
|
||||
tl.store(out_query_slot_mapping_ptr + query_idx, q_slot, mask=is_query)
|
||||
@@ -679,6 +682,9 @@ def _prepare_dflash_inputs_kernel(
|
||||
for i in range(q_pad_start, max_num_tokens, BLOCK_SIZE):
|
||||
block = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = block < max_num_tokens
|
||||
tl.store(out_input_ids_ptr + block, 0, mask=mask)
|
||||
tl.store(out_query_positions_ptr + block, 0, mask=mask)
|
||||
tl.store(out_is_padding_ptr + block, True, mask=mask)
|
||||
tl.store(out_query_slot_mapping_ptr + block, PAD_SLOT_ID, mask=mask)
|
||||
|
||||
|
||||
@@ -730,6 +736,7 @@ def prepare_dflash_inputs(
|
||||
_prepare_dflash_inputs_kernel[(num_reqs, num_blocks)](
|
||||
input_buffers.input_ids,
|
||||
input_buffers.positions,
|
||||
input_buffers.is_padding,
|
||||
input_buffers.query_start_loc,
|
||||
input_buffers.seq_lens,
|
||||
query_slot_mapping,
|
||||
|
||||
Reference in New Issue
Block a user