[Frontend] Require flag for loading text and image embeds (#27204)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
DarkLight1337
parent
db6f28d898
commit
58fab50d82
@@ -109,8 +109,7 @@ VLM_TEST_SETTINGS = {
|
||||
limit_mm_per_prompt={"image": 4},
|
||||
)
|
||||
],
|
||||
# TODO: Revert to "auto" when CPU backend can use torch > 2.6
|
||||
dtype="bfloat16" if current_platform.is_cpu() else "auto",
|
||||
vllm_runner_kwargs={"enable_mm_embeds": True},
|
||||
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
|
||||
),
|
||||
"qwen2_5_vl": VLMTestInfo(
|
||||
|
||||
@@ -292,6 +292,7 @@ def run_embedding_input_test(
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
distributed_executor_backend=distributed_executor_backend,
|
||||
default_torch_num_threads=1,
|
||||
enable_mm_embeds=True,
|
||||
) as vllm_model:
|
||||
outputs_per_case_for_original_input = [
|
||||
vllm_model.generate_greedy_logprobs(
|
||||
|
||||
@@ -34,6 +34,7 @@ def _run_test(
|
||||
dtype="half",
|
||||
enforce_eager=True,
|
||||
skip_tokenizer_init=True,
|
||||
enable_mm_embeds=True,
|
||||
# Limit the maximum number of sequences to avoid the
|
||||
# test going OOM during the warmup run
|
||||
max_num_seqs=32,
|
||||
|
||||
@@ -104,6 +104,11 @@ def can_initialize(
|
||||
m.setenv("VLLM_ATTENTION_BACKEND", "TRITON_ATTN")
|
||||
if model_arch == "WhisperForConditionalGeneration":
|
||||
m.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
||||
|
||||
extra_args = {}
|
||||
if model_arch in ("PrithviGeoSpatialMAE", "Terratorch"):
|
||||
extra_args["enable_mm_embeds"] = True
|
||||
|
||||
LLM(
|
||||
model_info.default,
|
||||
tokenizer=model_info.tokenizer,
|
||||
@@ -128,6 +133,7 @@ def can_initialize(
|
||||
else "vllm",
|
||||
hf_overrides=hf_overrides_fn,
|
||||
max_num_seqs=model_info.max_num_seqs,
|
||||
**extra_args,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -32,6 +32,7 @@ def test_inference(
|
||||
dtype="half",
|
||||
enforce_eager=True,
|
||||
skip_tokenizer_init=True,
|
||||
enable_mm_embeds=True,
|
||||
# Limit the maximum number of sequences to avoid the
|
||||
# test going OOM during the warmup run
|
||||
max_num_seqs=32,
|
||||
|
||||
Reference in New Issue
Block a user