[VLM] Merged multimodal processor for Qwen2-Audio (#11303)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
+23
-7
@@ -775,7 +775,7 @@ def get_dtype_size(dtype: torch.dtype) -> int:
|
||||
# `collections` helpers
|
||||
def is_list_of(
|
||||
value: object,
|
||||
typ: Type[T],
|
||||
typ: Union[type[T], tuple[type[T], ...]],
|
||||
*,
|
||||
check: Literal["first", "all"] = "first",
|
||||
) -> TypeIs[List[T]]:
|
||||
@@ -1282,6 +1282,7 @@ async def _run_task_with_lock(task: Callable, lock: asyncio.Lock, *args,
|
||||
def supports_kw(
|
||||
callable: Callable[..., object],
|
||||
kw_name: str,
|
||||
*,
|
||||
requires_kw_only: bool = False,
|
||||
allow_var_kwargs: bool = True,
|
||||
) -> bool:
|
||||
@@ -1326,6 +1327,8 @@ def resolve_mm_processor_kwargs(
|
||||
init_kwargs: Optional[Mapping[str, object]],
|
||||
inference_kwargs: Optional[Mapping[str, object]],
|
||||
callable: Callable[..., object],
|
||||
*,
|
||||
requires_kw_only: bool = True,
|
||||
allow_var_kwargs: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
"""Applies filtering to eliminate invalid mm_processor_kwargs, i.e.,
|
||||
@@ -1344,11 +1347,17 @@ def resolve_mm_processor_kwargs(
|
||||
runtime_mm_kwargs = get_allowed_kwarg_only_overrides(
|
||||
callable,
|
||||
overrides=inference_kwargs,
|
||||
allow_var_kwargs=allow_var_kwargs)
|
||||
requires_kw_only=requires_kw_only,
|
||||
allow_var_kwargs=allow_var_kwargs,
|
||||
)
|
||||
|
||||
# Filter init time multimodal processor kwargs provided
|
||||
init_mm_kwargs = get_allowed_kwarg_only_overrides(
|
||||
callable, overrides=init_kwargs, allow_var_kwargs=allow_var_kwargs)
|
||||
callable,
|
||||
overrides=init_kwargs,
|
||||
requires_kw_only=requires_kw_only,
|
||||
allow_var_kwargs=allow_var_kwargs,
|
||||
)
|
||||
|
||||
# Merge the final processor kwargs, prioritizing inference
|
||||
# time values over the initialization time values.
|
||||
@@ -1359,6 +1368,8 @@ def resolve_mm_processor_kwargs(
|
||||
def get_allowed_kwarg_only_overrides(
|
||||
callable: Callable[..., object],
|
||||
overrides: Optional[Mapping[str, object]],
|
||||
*,
|
||||
requires_kw_only: bool = True,
|
||||
allow_var_kwargs: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
@@ -1390,16 +1401,21 @@ def get_allowed_kwarg_only_overrides(
|
||||
for kwarg_name, val in overrides.items()
|
||||
if supports_kw(callable,
|
||||
kwarg_name,
|
||||
requires_kw_only=True,
|
||||
requires_kw_only=requires_kw_only,
|
||||
allow_var_kwargs=allow_var_kwargs)
|
||||
}
|
||||
|
||||
# If anything is dropped, log a warning
|
||||
dropped_keys = overrides.keys() - filtered_overrides.keys()
|
||||
if dropped_keys:
|
||||
logger.warning(
|
||||
"The following intended overrides are not keyword-only args "
|
||||
"and and will be dropped: %s", dropped_keys)
|
||||
if requires_kw_only:
|
||||
logger.warning(
|
||||
"The following intended overrides are not keyword-only args "
|
||||
"and and will be dropped: %s", dropped_keys)
|
||||
else:
|
||||
logger.warning(
|
||||
"The following intended overrides are not keyword args "
|
||||
"and and will be dropped: %s", dropped_keys)
|
||||
|
||||
return filtered_overrides
|
||||
|
||||
|
||||
Reference in New Issue
Block a user