[Multimodal] Optimize Qwen2/2.5-VL startup time (#19756)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu> Signed-off-by: Roger Wang <hey@rogerw.me> Co-authored-by: Roger Wang <hey@rogerw.me>
This commit is contained in:
co-authored by
Roger Wang
parent
caa680fd2e
commit
2c5302fadd
@@ -253,6 +253,26 @@ class MultiModalProfiler(Generic[_I]):
|
||||
seq_len: int,
|
||||
mm_counts: Optional[Mapping[str, int]] = None,
|
||||
) -> Mapping[str, int]:
|
||||
mm_inputs = self._get_dummy_mm_inputs(seq_len, mm_counts)
|
||||
max_tokens_per_item = self.processing_info.get_max_tokens_per_item(
|
||||
seq_len=seq_len, mm_counts=mm_counts)
|
||||
if max_tokens_per_item is not None:
|
||||
if mm_counts is None:
|
||||
total_mm_tokens = sum(max_tokens_per_item.values())
|
||||
else:
|
||||
total_mm_tokens = sum(max_tokens_per_item[k] * mm_counts[k]
|
||||
for k in max_tokens_per_item.keys()
|
||||
& mm_counts.keys())
|
||||
if total_mm_tokens > seq_len:
|
||||
logger.warning_once(
|
||||
"The sequence length (%d) is smaller than the pre-defined"
|
||||
" wosrt-case total number of multimodal tokens (%d). "
|
||||
"This may cause certain multi-modal inputs to fail during "
|
||||
"inference. To avoid this, you should increase "
|
||||
"`max_model_len` or reduce `mm_counts`.",
|
||||
seq_len,
|
||||
total_mm_tokens,
|
||||
)
|
||||
return max_tokens_per_item
|
||||
|
||||
mm_inputs = self._get_dummy_mm_inputs(seq_len, mm_counts)
|
||||
return self._get_mm_num_tokens(mm_inputs)
|
||||
|
||||
Reference in New Issue
Block a user