[Core] Increase default max_num_batched_tokens for multimodal models (#8028)

This commit is contained in:
Cyrus Leung
2024-08-30 08:20:34 -07:00
committed by GitHub
parent f97be32d1d
commit 98cef6a227
4 changed files with 33 additions and 12 deletions
+1
View File
@@ -921,6 +921,7 @@ class EngineArgs:
delay_factor=self.scheduler_delay_factor,
enable_chunked_prefill=self.enable_chunked_prefill,
embedding_mode=model_config.embedding_mode,
is_multimodal_model=model_config.is_multimodal_model,
preemption_mode=self.preemption_mode,
num_scheduler_steps=self.num_scheduler_steps,
send_delta_data=(envs.VLLM_USE_RAY_SPMD_WORKER