[MM] Remove text components from ProcessorInputs (#53093)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -107,7 +107,7 @@ def test_audio_chunk_counting(mock_ctx):
|
||||
mm_data = {"audio": [audio_1, audio_2]}
|
||||
prompt = "<|user|>Listen.<|end|>"
|
||||
|
||||
processed = processor._call_hf_processor(prompt, mm_data, {}, {})
|
||||
processed = processor._call_hf_processor(prompt, mm_data, {})
|
||||
|
||||
chunk_counts = processed["chunk_counts"]
|
||||
|
||||
|
||||
@@ -145,10 +145,10 @@ def get_transformers_backend_model_ids_to_test():
|
||||
)
|
||||
|
||||
|
||||
def get_text_token_prompts(
|
||||
def get_token_prompt(
|
||||
processor: BaseMultiModalProcessor,
|
||||
mm_data: MultiModalDataDict,
|
||||
):
|
||||
) -> list[int]:
|
||||
dummy_inputs = processor.dummy_inputs
|
||||
tokenizer: TokenizerLike = processor.info.get_tokenizer()
|
||||
model_config = processor.info.ctx.model_config
|
||||
@@ -178,21 +178,10 @@ def get_text_token_prompts(
|
||||
mm_options={},
|
||||
)
|
||||
|
||||
text_prompt: str | None
|
||||
token_prompt: list[int]
|
||||
if isinstance(inputs.prompt, list):
|
||||
text_prompt = None
|
||||
token_prompt = inputs.prompt
|
||||
elif isinstance(inputs.prompt, str):
|
||||
text_prompt = inputs.prompt
|
||||
token_prompt = tokenizer.encode(
|
||||
text_prompt,
|
||||
**processor.info.get_default_tok_params().get_encode_kwargs(),
|
||||
)
|
||||
else:
|
||||
if not isinstance(inputs.prompt, list):
|
||||
raise TypeError(type(inputs.prompt))
|
||||
|
||||
return text_prompt, token_prompt
|
||||
return inputs.prompt
|
||||
|
||||
|
||||
def random_vision_chunk(
|
||||
@@ -358,7 +347,7 @@ def _test_processing_correctness_one(
|
||||
):
|
||||
model_type = model_config.hf_config.model_type
|
||||
|
||||
text_prompt, token_prompt = get_text_token_prompts(baseline_processor, mm_data)
|
||||
token_prompt = get_token_prompt(baseline_processor, mm_data)
|
||||
mm_items = baseline_processor.info.parse_mm_data(mm_data)
|
||||
ignore_mm_keys = _IGNORE_MM_KEYS.get(model_type, set[str]())
|
||||
|
||||
@@ -381,55 +370,10 @@ def _test_processing_correctness_one(
|
||||
msg=(
|
||||
f"Failed ({batch_idx=}, {hit_rate=}, "
|
||||
f"{num_batches=}, {simplify_rate=}, "
|
||||
f"{text_prompt=}, {token_prompt=}, {mm_data=})"
|
||||
f"{token_prompt=}, {mm_data=})"
|
||||
),
|
||||
)
|
||||
|
||||
if text_prompt is not None:
|
||||
baseline_text_result = baseline_processor(
|
||||
text_prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
cached_text_result = cached_processor(
|
||||
text_prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
_assert_inputs_equal(
|
||||
baseline_text_result,
|
||||
cached_text_result,
|
||||
ignore_mm_keys=ignore_mm_keys,
|
||||
msg=(
|
||||
f"Failed ({batch_idx=}, {hit_rate=}, "
|
||||
f"{num_batches=}, {simplify_rate=}, "
|
||||
f"{text_prompt=}, {token_prompt=}, {mm_data=})"
|
||||
),
|
||||
)
|
||||
|
||||
_assert_inputs_equal(
|
||||
baseline_text_result,
|
||||
baseline_tokenized_result,
|
||||
ignore_mm_keys=ignore_mm_keys,
|
||||
msg=(
|
||||
f"Failed ({batch_idx=}, {hit_rate=}, "
|
||||
f"{num_batches=}, {simplify_rate=}, "
|
||||
f"{text_prompt=}, {token_prompt=}, {mm_data=})"
|
||||
),
|
||||
)
|
||||
|
||||
_assert_inputs_equal(
|
||||
cached_text_result,
|
||||
cached_tokenized_result,
|
||||
ignore_mm_keys=ignore_mm_keys,
|
||||
msg=(
|
||||
f"Failed ({batch_idx=}, {hit_rate=}, "
|
||||
f"{num_batches=}, {simplify_rate=}, "
|
||||
f"{text_prompt=}, {token_prompt=}, {mm_data=})"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", get_model_ids_to_test())
|
||||
@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
|
||||
|
||||
@@ -11,6 +11,7 @@ from pqdm.threads import pqdm
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.parse import ImageSize
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor
|
||||
from vllm.tokenizers.hf import maybe_make_thread_pool
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
@@ -144,7 +145,14 @@ def test_processor_prompt_replacements_regression(model_id, num_imgs):
|
||||
mm_processor_kwargs=None,
|
||||
limit_mm_per_prompt={"image": num_imgs},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
# Avoid tokenizer already borrowed error
|
||||
maybe_make_thread_pool(ctx.tokenizer)
|
||||
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ctx.model_config,
|
||||
tokenizer=ctx.tokenizer,
|
||||
)
|
||||
|
||||
image_ratios = [
|
||||
(171, 152),
|
||||
@@ -177,7 +185,14 @@ def test_processor_prompt_replacements_all(model_id, num_imgs):
|
||||
mm_processor_kwargs=None,
|
||||
limit_mm_per_prompt={"image": num_imgs},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
# Avoid tokenizer already borrowed error
|
||||
maybe_make_thread_pool(ctx.tokenizer)
|
||||
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ctx.model_config,
|
||||
tokenizer=ctx.tokenizer,
|
||||
)
|
||||
|
||||
seen_aspect_ratios = set[float]()
|
||||
image_sizes = list[ImageSize]()
|
||||
|
||||
@@ -11,6 +11,7 @@ from pqdm.threads import pqdm
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.parse import ImageSize
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor
|
||||
from vllm.tokenizers.hf import maybe_make_thread_pool
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
@@ -42,7 +43,15 @@ def test_processor_max_tokens(model_id):
|
||||
mm_processor_kwargs=None,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
# Avoid tokenizer already borrowed error
|
||||
maybe_make_thread_pool(ctx.tokenizer)
|
||||
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ctx.model_config,
|
||||
tokenizer=ctx.tokenizer,
|
||||
)
|
||||
|
||||
info = processor.info
|
||||
|
||||
seen_aspect_ratios = set[float]()
|
||||
@@ -142,7 +151,14 @@ def test_processor_prompt_replacements_regression(model_id, num_imgs):
|
||||
mm_processor_kwargs=None,
|
||||
limit_mm_per_prompt={"image": num_imgs},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
# Avoid tokenizer already borrowed error
|
||||
maybe_make_thread_pool(ctx.tokenizer)
|
||||
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ctx.model_config,
|
||||
tokenizer=ctx.tokenizer,
|
||||
)
|
||||
|
||||
image_ratios = [
|
||||
(171, 152),
|
||||
|
||||
@@ -35,12 +35,10 @@ from vllm.sequence import IntermediateTensors
|
||||
|
||||
|
||||
class _Tokenizer:
|
||||
def encode(self, text, add_special_tokens=False):
|
||||
del add_special_tokens
|
||||
def encode(self, text, **kwargs):
|
||||
return [ord(char) for char in text]
|
||||
|
||||
def decode(self, token_ids, **kwargs):
|
||||
del kwargs
|
||||
return "".join(chr(token_id) for token_id in token_ids)
|
||||
|
||||
def batch_decode(self, batch_token_ids, **kwargs):
|
||||
|
||||
@@ -185,11 +185,6 @@ def test_openvla_prompt_update_inserts_image_tokens_after_bos() -> None:
|
||||
image = Image.new("RGB", (640, 480), color=(255, 255, 255))
|
||||
mm_items = MultiModalDataItems({"image": ImageProcessorItems([image])})
|
||||
|
||||
assert (
|
||||
processor._hf_processor_applies_updates("In: test\nOut:", mm_items, {}, {})
|
||||
is False
|
||||
)
|
||||
|
||||
prompt_update = processor._get_prompt_updates(mm_items, {}, {})[0]
|
||||
resolved = prompt_update.resolve(0)
|
||||
content = resolved.content
|
||||
|
||||
@@ -37,7 +37,7 @@ from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
from ....utils import create_new_process_for_each_test
|
||||
from ...registry import HF_EXAMPLE_MODELS
|
||||
from ...utils import dummy_hf_overrides
|
||||
from .test_common import get_model_ids_to_test, get_text_token_prompts
|
||||
from .test_common import get_model_ids_to_test, get_token_prompt
|
||||
|
||||
ImageInput = list[Image.Image]
|
||||
VideoInput: TypeAlias = (
|
||||
@@ -107,10 +107,10 @@ def create_batched_mm_kwargs(
|
||||
}
|
||||
|
||||
# video metadata will be added back to the resized video data here.
|
||||
text_prompt, token_prompt = get_text_token_prompts(processor, resized_mm_data)
|
||||
token_prompt = get_token_prompt(processor, resized_mm_data)
|
||||
|
||||
mm_kwargs = processor(
|
||||
prompt=token_prompt if text_prompt is None else text_prompt,
|
||||
prompt=token_prompt,
|
||||
mm_items=processor.info.parse_mm_data(resized_mm_data),
|
||||
hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
|
||||
)["mm_kwargs"].require_data()
|
||||
|
||||
Reference in New Issue
Block a user