[Bugfix][Bench] Fix bench mm-processor crash in shared request sampling (#56300)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>

tests/benchmarks/test_throughput_cli.py

@@ -17,6 +17,7 @@ from transformers import AutoTokenizer, PreTrainedTokenizerBase
 from vllm.benchmarks.datasets import SampleRequest
 from vllm.benchmarks.throughput import (
     _run_vllm_chat_requests,
+    _to_serve_args,
     add_cli_args,
     assign_loras,
     get_requests,
@@ -277,3 +278,91 @@ def test_get_requests_allows_multimodal_on_plain_vllm_backend(
     requests = get_requests(args, hf_tokenizer)
     assert len(requests) == 3
     assert all(isinstance(r, SampleRequest) for r in requests)
+
+
+def test_to_serve_args_supports_mm_processor_flags() -> None:
+    """The adapter must serve callers without throughput's legacy flags.
+
+    ``bench mm-processor`` exposes neither --input-len/--output-len/--prefix-len
+    nor --backend, and leaves --random-prefix-len at its default of 0. The
+    adapter must not read the legacy flags unconditionally, and must keep the
+    explicit 0 instead of coercing it to None (which later crashed
+    RandomMultiModalDataset.get_prefix on ``None <= 0``).
+    """
+    args = SimpleNamespace(
+        random_input_len=32,
+        random_output_len=1,
+        random_prefix_len=0,
+    )
+    serve_args = _to_serve_args(args)
+
+    assert serve_args.random_input_len == 32
+    assert serve_args.random_output_len == 1
+    assert serve_args.random_prefix_len == 0
+    assert serve_args.backend == "vllm-chat"
+    assert serve_args.enable_multimodal_chat
+    assert serve_args.no_oversample is False
+
+
+def test_to_serve_args_keeps_legacy_flag_fallback() -> None:
+    """Callers that only set the legacy flags still get them mapped."""
+    args = SimpleNamespace(
+        input_len=256,
+        output_len=8,
+        prefix_len=16,
+        backend="vllm",
+    )
+    serve_args = _to_serve_args(args)
+
+    assert serve_args.random_input_len == 256
+    assert serve_args.random_output_len == 8
+    assert serve_args.random_prefix_len == 16
+    assert serve_args.backend == "vllm"
+    assert not serve_args.enable_multimodal_chat
+
+
+@pytest.mark.benchmark
+def test_get_requests_random_mm_with_mm_processor_args(
+    hf_tokenizer: PreTrainedTokenizerBase,
+) -> None:
+    """Regression: bench mm-processor's random-mm sampling must not crash.
+
+    get_requests feeds mm-processor's parsed args (no legacy length/backend
+    flags) through the shared get_samples dispatch. With the default
+    --random-prefix-len 0 this previously crashed inside
+    RandomMultiModalDataset.get_prefix with a TypeError.
+    """
+    from vllm.benchmarks.mm_processor import add_cli_args as add_mm_cli_args
+
+    parser = FlexibleArgumentParser()
+    add_mm_cli_args(parser)
+    args = parser.parse_args(
+        [
+            "--dataset-name",
+            "random-mm",
+            "--num-prompts",
+            "3",
+            "--random-input-len",
+            "32",
+            "--random-output-len",
+            "1",
+            "--random-mm-base-items-per-request",
+            "1",
+            "--random-mm-num-mm-items-range-ratio",
+            "0",
+            "--random-mm-limit-mm-per-prompt",
+            '{"image": 1}',
+            "--random-mm-bucket-config",
+            "{(224, 224, 1): 1.0}",
+        ]
+    )
+
+    requests = get_requests(args, hf_tokenizer)
+    assert len(requests) == 3
+    assert all(isinstance(r, SampleRequest) for r in requests)
+    # mm-processor drives llm.chat, so prompts must be chat-formatted with
+    # multimodal content attached.
+    for req in requests:
+        assert isinstance(req.prompt, list)
+        assert req.prompt[0]["role"] == "user"
+        assert any(item.get("type") == "image_url" for item in req.prompt[0]["content"])

vllm/benchmarks/throughput.py

@@ -539,27 +539,38 @@ def _to_serve_args(args: argparse.Namespace) -> argparse.Namespace:
     """
     d = vars(args).copy()
     # random_*: prefer --random-* over legacy --input/output/prefix-len.
-    d["random_input_len"] = getattr(args, "random_input_len", None) or args.input_len
-    d["random_output_len"] = getattr(args, "random_output_len", None) or args.output_len
-    d["random_prefix_len"] = getattr(args, "random_prefix_len", None) or args.prefix_len
+    # Callers without the legacy flags (e.g. bench mm-processor) omit them, so
+    # read them optionally. Identity checks, not truthiness: an explicit 0
+    # (the default of --random-prefix-len) must not fall through to them.
+    input_len = getattr(args, "input_len", None)
+    output_len = getattr(args, "output_len", None)
+    prefix_len = getattr(args, "prefix_len", None)
+    d["random_input_len"] = getattr(args, "random_input_len", input_len)
+    d["random_output_len"] = getattr(args, "random_output_len", output_len)
+    d["random_prefix_len"] = getattr(args, "random_prefix_len", prefix_len)
     # --output-len maps to the per-dataset output-len entry points get_samples
     # reads. None passes through (each dataset applies its own default),
     # matching prior throughput behaviour of omitting output_len when unset.
-    d["hf_output_len"] = args.output_len
-    d["sharegpt_output_len"] = args.output_len
+    d["hf_output_len"] = output_len
+    d["sharegpt_output_len"] = output_len
     # sonnet reads dedicated attrs; fall back to SonnetDataset's own defaults.
-    d["sonnet_input_len"] = args.input_len if args.input_len is not None else 550
-    d["sonnet_output_len"] = args.output_len if args.output_len is not None else 150
-    d["sonnet_prefix_len"] = args.prefix_len
+    d["sonnet_input_len"] = input_len if input_len is not None else 550
+    d["sonnet_output_len"] = output_len if output_len is not None else 150
+    d["sonnet_prefix_len"] = prefix_len
     # Explicit --enable-multimodal-chat wins; otherwise auto-enable for the
-    # multimodal chat backend (preserves today's vllm-chat handling).
+    # multimodal chat backend (preserves today's vllm-chat handling). Callers
+    # without a --backend flag (e.g. bench mm-processor) drive every request
+    # through llm.chat, so default to the vllm-chat behaviour.
+    backend = getattr(args, "backend", "vllm-chat")
+    d["backend"] = backend
     d["enable_multimodal_chat"] = bool(
-        getattr(args, "enable_multimodal_chat", False) or args.backend == "vllm-chat"
+        getattr(args, "enable_multimodal_chat", False) or backend == "vllm-chat"
     )
     # serve-only attrs throughput never exposed; keep serve's defaults.
     d.setdefault("disable_shuffle", False)
     d.setdefault("skip_chat_template", False)
     d.setdefault("no_stream", False)
+    d.setdefault("no_oversample", False)
     d.setdefault("request_id_prefix", "")
     d.setdefault("chat_template_kwargs", None)
     return argparse.Namespace(**d)