[BugFix] Support async scheduling with prompt embeds for multimodal models (#45673)

Signed-off-by: Ruinan Ma <r7ma3088@gmail.com>
This commit is contained in:
Ruinan Ma
2026-06-16 04:12:54 +00:00
committed by GitHub
parent e3cfea2e1b
commit 2addbb9cc9
2 changed files with 5 additions and 20 deletions
+5 -1
View File
@@ -1779,13 +1779,17 @@ class GPUModelRunner(
num_common_tokens = len(sample_flattened_indices)
total_without_spec = total_num_scheduled_tokens - total_num_spec_tokens
if self.enable_prompt_embeds:
# The multimodal embed path reads is_token_ids.gpu; its .cpu copy is
# refreshed every step but the async fast paths below only scatter
# input_ids.gpu, so refresh is_token_ids.gpu here too.
self.is_token_ids.copy_to_gpu(total_num_scheduled_tokens)
if num_common_tokens < total_without_spec:
# If not all requests are decodes from the last iteration,
# we need to copy the input_ids_cpu to the GPU first.
self.input_ids.copy_to_gpu(total_num_scheduled_tokens)
if self.enable_prompt_embeds:
self.inputs_embeds.copy_to_gpu(total_num_scheduled_tokens)
self.is_token_ids.copy_to_gpu(total_num_scheduled_tokens)
if num_common_tokens == 0:
# No requests in common with the previous iteration
# So input_ids.cpu will have all the input ids.