[BugFix] Support async scheduling with prompt embeds for multimodal models (#45673)
Signed-off-by: Ruinan Ma <r7ma3088@gmail.com>
This commit is contained in:
@@ -1779,13 +1779,17 @@ class GPUModelRunner(
|
||||
|
||||
num_common_tokens = len(sample_flattened_indices)
|
||||
total_without_spec = total_num_scheduled_tokens - total_num_spec_tokens
|
||||
if self.enable_prompt_embeds:
|
||||
# The multimodal embed path reads is_token_ids.gpu; its .cpu copy is
|
||||
# refreshed every step but the async fast paths below only scatter
|
||||
# input_ids.gpu, so refresh is_token_ids.gpu here too.
|
||||
self.is_token_ids.copy_to_gpu(total_num_scheduled_tokens)
|
||||
if num_common_tokens < total_without_spec:
|
||||
# If not all requests are decodes from the last iteration,
|
||||
# we need to copy the input_ids_cpu to the GPU first.
|
||||
self.input_ids.copy_to_gpu(total_num_scheduled_tokens)
|
||||
if self.enable_prompt_embeds:
|
||||
self.inputs_embeds.copy_to_gpu(total_num_scheduled_tokens)
|
||||
self.is_token_ids.copy_to_gpu(total_num_scheduled_tokens)
|
||||
if num_common_tokens == 0:
|
||||
# No requests in common with the previous iteration
|
||||
# So input_ids.cpu will have all the input ids.
|
||||
|
||||
Reference in New Issue
Block a user