add enqueue all option to throughput benchmark (#42975)

Signed-off-by: Philip Maybank <pmaybank@amd.com>
Signed-off-by: pmaybank <113125070+pmaybank@users.noreply.github.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
pmaybank
2026-05-19 20:16:02 -07:00
committed by GitHub
co-authored by gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent c628a93a64
commit fadf5d332c
3 changed files with 217 additions and 7 deletions
@@ -76,6 +76,51 @@ def test_chat_forwards_mm_processor_kwargs() -> None:
)
def test_enqueue_chat_forwards_mm_processor_kwargs() -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"do_pan_and_scan": True}
sampling_params = SamplingParams(max_tokens=1)
messages = [{"role": "user", "content": "hello"}]
llm._add_chat_requests = Mock(return_value=["req-0"])
request_ids = llm.enqueue_chat(
messages,
sampling_params=sampling_params,
use_tqdm=False,
mm_processor_kwargs=mm_processor_kwargs,
)
assert request_ids == ["req-0"]
assert llm._add_chat_requests.call_args.kwargs["mm_processor_kwargs"] == (
mm_processor_kwargs
)
def test_run_chat_forwards_mm_processor_kwargs() -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"num_crops": 8}
sampling_params = SamplingParams(max_tokens=1)
messages = [{"role": "user", "content": "hello"}]
sentinel_output = ["done"]
llm._add_chat_requests = Mock()
llm._run_engine = Mock(return_value=sentinel_output)
outputs = llm._run_chat(
messages=messages,
params=sampling_params,
output_type=object,
use_tqdm=False,
mm_processor_kwargs=mm_processor_kwargs,
)
assert outputs == sentinel_output
assert llm._add_chat_requests.call_args.kwargs["mm_processor_kwargs"] == (
mm_processor_kwargs
)
def test_run_completion_forwards_mm_processor_kwargs() -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"min_pixels": 4 * 28 * 28}