[MM] Remove text components from ProcessorInputs (#53093)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-08-20 20:42:53 -07:00
committed by GitHub
parent 2adf4b9e2a
commit b00f475f09
92 changed files with 220 additions and 806 deletions
@@ -107,7 +107,7 @@ def test_audio_chunk_counting(mock_ctx):
mm_data = {"audio": [audio_1, audio_2]}
prompt = "<|user|>Listen.<|end|>"
processed = processor._call_hf_processor(prompt, mm_data, {}, {})
processed = processor._call_hf_processor(prompt, mm_data, {})
chunk_counts = processed["chunk_counts"]
@@ -145,10 +145,10 @@ def get_transformers_backend_model_ids_to_test():
)
def get_text_token_prompts(
def get_token_prompt(
processor: BaseMultiModalProcessor,
mm_data: MultiModalDataDict,
):
) -> list[int]:
dummy_inputs = processor.dummy_inputs
tokenizer: TokenizerLike = processor.info.get_tokenizer()
model_config = processor.info.ctx.model_config
@@ -178,21 +178,10 @@ def get_text_token_prompts(
mm_options={},
)
text_prompt: str | None
token_prompt: list[int]
if isinstance(inputs.prompt, list):
text_prompt = None
token_prompt = inputs.prompt
elif isinstance(inputs.prompt, str):
text_prompt = inputs.prompt
token_prompt = tokenizer.encode(
text_prompt,
**processor.info.get_default_tok_params().get_encode_kwargs(),
)
else:
if not isinstance(inputs.prompt, list):
raise TypeError(type(inputs.prompt))
return text_prompt, token_prompt
return inputs.prompt
def random_vision_chunk(
@@ -358,7 +347,7 @@ def _test_processing_correctness_one(
):
model_type = model_config.hf_config.model_type
text_prompt, token_prompt = get_text_token_prompts(baseline_processor, mm_data)
token_prompt = get_token_prompt(baseline_processor, mm_data)
mm_items = baseline_processor.info.parse_mm_data(mm_data)
ignore_mm_keys = _IGNORE_MM_KEYS.get(model_type, set[str]())
@@ -381,55 +370,10 @@ def _test_processing_correctness_one(
msg=(
f"Failed ({batch_idx=}, {hit_rate=}, "
f"{num_batches=}, {simplify_rate=}, "
f"{text_prompt=}, {token_prompt=}, {mm_data=})"
f"{token_prompt=}, {mm_data=})"
),
)
if text_prompt is not None:
baseline_text_result = baseline_processor(
text_prompt,
mm_items=mm_items,
hf_processor_mm_kwargs={},
)
cached_text_result = cached_processor(
text_prompt,
mm_items=mm_items,
hf_processor_mm_kwargs={},
)
_assert_inputs_equal(
baseline_text_result,
cached_text_result,
ignore_mm_keys=ignore_mm_keys,
msg=(
f"Failed ({batch_idx=}, {hit_rate=}, "
f"{num_batches=}, {simplify_rate=}, "
f"{text_prompt=}, {token_prompt=}, {mm_data=})"
),
)
_assert_inputs_equal(
baseline_text_result,
baseline_tokenized_result,
ignore_mm_keys=ignore_mm_keys,
msg=(
f"Failed ({batch_idx=}, {hit_rate=}, "
f"{num_batches=}, {simplify_rate=}, "
f"{text_prompt=}, {token_prompt=}, {mm_data=})"
),
)
_assert_inputs_equal(
cached_text_result,
cached_tokenized_result,
ignore_mm_keys=ignore_mm_keys,
msg=(
f"Failed ({batch_idx=}, {hit_rate=}, "
f"{num_batches=}, {simplify_rate=}, "
f"{text_prompt=}, {token_prompt=}, {mm_data=})"
),
)
@pytest.mark.parametrize("model_id", get_model_ids_to_test())
@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
@@ -11,6 +11,7 @@ from pqdm.threads import pqdm
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.parse import ImageSize
from vllm.multimodal.processing import BaseMultiModalProcessor
from vllm.tokenizers.hf import maybe_make_thread_pool
from ...utils import build_model_context
@@ -144,7 +145,14 @@ def test_processor_prompt_replacements_regression(model_id, num_imgs):
mm_processor_kwargs=None,
limit_mm_per_prompt={"image": num_imgs},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
# Avoid tokenizer already borrowed error
maybe_make_thread_pool(ctx.tokenizer)
processor = MULTIMODAL_REGISTRY.create_processor(
ctx.model_config,
tokenizer=ctx.tokenizer,
)
image_ratios = [
(171, 152),
@@ -177,7 +185,14 @@ def test_processor_prompt_replacements_all(model_id, num_imgs):
mm_processor_kwargs=None,
limit_mm_per_prompt={"image": num_imgs},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
# Avoid tokenizer already borrowed error
maybe_make_thread_pool(ctx.tokenizer)
processor = MULTIMODAL_REGISTRY.create_processor(
ctx.model_config,
tokenizer=ctx.tokenizer,
)
seen_aspect_ratios = set[float]()
image_sizes = list[ImageSize]()
@@ -11,6 +11,7 @@ from pqdm.threads import pqdm
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.parse import ImageSize
from vllm.multimodal.processing import BaseMultiModalProcessor
from vllm.tokenizers.hf import maybe_make_thread_pool
from ...utils import build_model_context
@@ -42,7 +43,15 @@ def test_processor_max_tokens(model_id):
mm_processor_kwargs=None,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
# Avoid tokenizer already borrowed error
maybe_make_thread_pool(ctx.tokenizer)
processor = MULTIMODAL_REGISTRY.create_processor(
ctx.model_config,
tokenizer=ctx.tokenizer,
)
info = processor.info
seen_aspect_ratios = set[float]()
@@ -142,7 +151,14 @@ def test_processor_prompt_replacements_regression(model_id, num_imgs):
mm_processor_kwargs=None,
limit_mm_per_prompt={"image": num_imgs},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
# Avoid tokenizer already borrowed error
maybe_make_thread_pool(ctx.tokenizer)
processor = MULTIMODAL_REGISTRY.create_processor(
ctx.model_config,
tokenizer=ctx.tokenizer,
)
image_ratios = [
(171, 152),
@@ -35,12 +35,10 @@ from vllm.sequence import IntermediateTensors
class _Tokenizer:
def encode(self, text, add_special_tokens=False):
del add_special_tokens
def encode(self, text, **kwargs):
return [ord(char) for char in text]
def decode(self, token_ids, **kwargs):
del kwargs
return "".join(chr(token_id) for token_id in token_ids)
def batch_decode(self, batch_token_ids, **kwargs):
@@ -185,11 +185,6 @@ def test_openvla_prompt_update_inserts_image_tokens_after_bos() -> None:
image = Image.new("RGB", (640, 480), color=(255, 255, 255))
mm_items = MultiModalDataItems({"image": ImageProcessorItems([image])})
assert (
processor._hf_processor_applies_updates("In: test\nOut:", mm_items, {}, {})
is False
)
prompt_update = processor._get_prompt_updates(mm_items, {}, {})[0]
resolved = prompt_update.resolve(0)
content = resolved.content
@@ -37,7 +37,7 @@ from vllm.utils.torch_utils import set_default_torch_dtype
from ....utils import create_new_process_for_each_test
from ...registry import HF_EXAMPLE_MODELS
from ...utils import dummy_hf_overrides
from .test_common import get_model_ids_to_test, get_text_token_prompts
from .test_common import get_model_ids_to_test, get_token_prompt
ImageInput = list[Image.Image]
VideoInput: TypeAlias = (
@@ -107,10 +107,10 @@ def create_batched_mm_kwargs(
}
# video metadata will be added back to the resized video data here.
text_prompt, token_prompt = get_text_token_prompts(processor, resized_mm_data)
token_prompt = get_token_prompt(processor, resized_mm_data)
mm_kwargs = processor(
prompt=token_prompt if text_prompt is None else text_prompt,
prompt=token_prompt,
mm_items=processor.info.parse_mm_data(resized_mm_data),
hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
)["mm_kwargs"].require_data()