[BugFix] Support async scheduling with prompt embeds for multimodal models (#45673)
Signed-off-by: Ruinan Ma <r7ma3088@gmail.com>
This commit is contained in:
@@ -966,15 +966,6 @@ class VllmConfig:
|
|||||||
"Async scheduling is not compatible with "
|
"Async scheduling is not compatible with "
|
||||||
"disable_padded_drafter_batch=True."
|
"disable_padded_drafter_batch=True."
|
||||||
)
|
)
|
||||||
if (
|
|
||||||
self.model_config is not None
|
|
||||||
and self.model_config.enable_prompt_embeds
|
|
||||||
and self.model_config.is_multimodal_model
|
|
||||||
):
|
|
||||||
raise ValueError(
|
|
||||||
"Async scheduling is not yet supported with prompt embeds "
|
|
||||||
"for multimodal models."
|
|
||||||
)
|
|
||||||
if not executor_supports_async_sched:
|
if not executor_supports_async_sched:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"`{executor_backend}` does not support async scheduling yet."
|
f"`{executor_backend}` does not support async scheduling yet."
|
||||||
@@ -1018,16 +1009,6 @@ class VllmConfig:
|
|||||||
executor_backend,
|
executor_backend,
|
||||||
)
|
)
|
||||||
self.scheduler_config.async_scheduling = False
|
self.scheduler_config.async_scheduling = False
|
||||||
elif (
|
|
||||||
self.model_config is not None
|
|
||||||
and self.model_config.enable_prompt_embeds
|
|
||||||
and self.model_config.is_multimodal_model
|
|
||||||
):
|
|
||||||
logger.warning_once(
|
|
||||||
"Async scheduling is not yet supported with prompt embeds "
|
|
||||||
"for multimodal models and will be disabled."
|
|
||||||
)
|
|
||||||
self.scheduler_config.async_scheduling = False
|
|
||||||
else:
|
else:
|
||||||
self.scheduler_config.async_scheduling = True
|
self.scheduler_config.async_scheduling = True
|
||||||
|
|
||||||
|
|||||||
@@ -1779,13 +1779,17 @@ class GPUModelRunner(
|
|||||||
|
|
||||||
num_common_tokens = len(sample_flattened_indices)
|
num_common_tokens = len(sample_flattened_indices)
|
||||||
total_without_spec = total_num_scheduled_tokens - total_num_spec_tokens
|
total_without_spec = total_num_scheduled_tokens - total_num_spec_tokens
|
||||||
|
if self.enable_prompt_embeds:
|
||||||
|
# The multimodal embed path reads is_token_ids.gpu; its .cpu copy is
|
||||||
|
# refreshed every step but the async fast paths below only scatter
|
||||||
|
# input_ids.gpu, so refresh is_token_ids.gpu here too.
|
||||||
|
self.is_token_ids.copy_to_gpu(total_num_scheduled_tokens)
|
||||||
if num_common_tokens < total_without_spec:
|
if num_common_tokens < total_without_spec:
|
||||||
# If not all requests are decodes from the last iteration,
|
# If not all requests are decodes from the last iteration,
|
||||||
# we need to copy the input_ids_cpu to the GPU first.
|
# we need to copy the input_ids_cpu to the GPU first.
|
||||||
self.input_ids.copy_to_gpu(total_num_scheduled_tokens)
|
self.input_ids.copy_to_gpu(total_num_scheduled_tokens)
|
||||||
if self.enable_prompt_embeds:
|
if self.enable_prompt_embeds:
|
||||||
self.inputs_embeds.copy_to_gpu(total_num_scheduled_tokens)
|
self.inputs_embeds.copy_to_gpu(total_num_scheduled_tokens)
|
||||||
self.is_token_ids.copy_to_gpu(total_num_scheduled_tokens)
|
|
||||||
if num_common_tokens == 0:
|
if num_common_tokens == 0:
|
||||||
# No requests in common with the previous iteration
|
# No requests in common with the previous iteration
|
||||||
# So input_ids.cpu will have all the input ids.
|
# So input_ids.cpu will have all the input ids.
|
||||||
|
|||||||
Reference in New Issue
Block a user