[ModelRunner V2] Enable by default for all dense models (#44443)
Signed-off-by: yewentao256 <zhyanwentao@126.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -18,9 +18,7 @@ steps:
|
||||
- set -x
|
||||
- export VLLM_USE_V2_MODEL_RUNNER=1
|
||||
- pytest -v -s v1/engine/test_llm_engine.py -k "not test_engine_metrics"
|
||||
# This requires eager until we sort out CG correctness issues.
|
||||
# TODO: remove ENFORCE_EAGER here after https://github.com/vllm-project/vllm/pull/32936 is merged.
|
||||
- ENFORCE_EAGER=1 pytest -v -s v1/e2e/general/test_async_scheduling.py -k "not ngram"
|
||||
- pytest -v -s v1/e2e/general/test_async_scheduling.py -k "not ngram"
|
||||
- pytest -v -s v1/e2e/general/test_context_length.py
|
||||
- pytest -v -s v1/e2e/general/test_min_tokens.py
|
||||
# Temporary hack filter to exclude ngram spec decoding based tests.
|
||||
|
||||
@@ -283,9 +283,14 @@ def test_multiproc_executor_pipeline_parallel():
|
||||
output_rank = executor._get_output_rank()
|
||||
assert output_rank == 2, "Output rank should be 2 (first rank of last PP stage)"
|
||||
|
||||
# Verify max_concurrent_batches for pipeline parallel
|
||||
assert vllm_config.max_concurrent_batches == 2, (
|
||||
"Max concurrent batches should equal PP size"
|
||||
# V2 model runner uses one extra batch to overlap async scheduling.
|
||||
expected_concurrent_batches = 2 + int(
|
||||
vllm_config.scheduler_config.async_scheduling
|
||||
and vllm_config.use_v2_model_runner
|
||||
)
|
||||
assert vllm_config.max_concurrent_batches == expected_concurrent_batches, (
|
||||
"Max concurrent batches should follow the configured PP/async "
|
||||
"scheduling policy"
|
||||
)
|
||||
|
||||
finally:
|
||||
|
||||
@@ -84,7 +84,13 @@ def assert_executor(executor, tp_size, pp_size):
|
||||
assert executor._get_output_rank() == expected_output_rank
|
||||
|
||||
if pp_size > 1:
|
||||
assert executor.vllm_config.max_concurrent_batches == pp_size
|
||||
expected_concurrent_batches = pp_size + int(
|
||||
executor.vllm_config.scheduler_config.async_scheduling
|
||||
and executor.vllm_config.use_v2_model_runner
|
||||
)
|
||||
assert (
|
||||
executor.vllm_config.max_concurrent_batches == expected_concurrent_batches
|
||||
)
|
||||
|
||||
executor.check_health()
|
||||
assert not executor.is_failed
|
||||
|
||||
+23
-1
@@ -118,7 +118,17 @@ def test_v2_model_runner_env_tri_state(monkeypatch, env_value, expected):
|
||||
is_moe=False,
|
||||
is_quantized=False,
|
||||
),
|
||||
False,
|
||||
True,
|
||||
),
|
||||
(
|
||||
SimpleNamespace(
|
||||
model="google/gemma-2-2b",
|
||||
architectures=["Gemma2ForCausalLM"],
|
||||
runner_type="generate",
|
||||
is_moe=False,
|
||||
is_quantized=False,
|
||||
),
|
||||
True,
|
||||
),
|
||||
(
|
||||
SimpleNamespace(
|
||||
@@ -197,6 +207,18 @@ def test_v2_model_runner_env_tri_state(monkeypatch, env_value, expected):
|
||||
runner_type="generate",
|
||||
is_moe=False,
|
||||
is_quantized=False,
|
||||
is_hybrid=True,
|
||||
),
|
||||
False,
|
||||
),
|
||||
(
|
||||
SimpleNamespace(
|
||||
model="state-spaces/mamba-130m-hf",
|
||||
architectures=["MambaForCausalLM"],
|
||||
runner_type="generate",
|
||||
is_moe=False,
|
||||
is_quantized=False,
|
||||
is_attention_free=True,
|
||||
),
|
||||
False,
|
||||
),
|
||||
|
||||
@@ -478,10 +478,14 @@ def test_cannot_schedule_after_recv():
|
||||
# request is retrieved from preempted list.
|
||||
scheduler_output = scheduler.schedule()
|
||||
model_runner_output = create_model_runner_output(reqs=[request_remote])
|
||||
assert (
|
||||
scheduler_output.scheduled_cached_reqs.num_computed_tokens[0]
|
||||
== NUM_PROMPT_BLOCKS * BLOCK_SIZE
|
||||
)
|
||||
# V2 emits a resumed (previously preempted) request as a NewRequestData
|
||||
# rather than a cached request.
|
||||
if scheduler.use_v2_model_runner:
|
||||
num_computed = scheduler_output.scheduled_new_reqs[0].num_computed_tokens
|
||||
else:
|
||||
cached = scheduler_output.scheduled_cached_reqs
|
||||
num_computed = cached.num_computed_tokens[0]
|
||||
assert num_computed == NUM_PROMPT_BLOCKS * BLOCK_SIZE
|
||||
scheduler.update_from_output(scheduler_output, model_runner_output)
|
||||
assert len(scheduler.running) == 1
|
||||
assert _num_waiting_requests(scheduler) == 0
|
||||
|
||||
+8
-5
@@ -67,12 +67,9 @@ logger = init_logger(__name__)
|
||||
|
||||
DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES = frozenset(
|
||||
{
|
||||
"Qwen3ForCausalLM",
|
||||
"DeepseekV2ForCausalLM",
|
||||
"Qwen2MoeForCausalLM",
|
||||
"GraniteMoeForCausalLM",
|
||||
"LlamaForCausalLM",
|
||||
"MistralForCausalLM",
|
||||
}
|
||||
)
|
||||
|
||||
@@ -565,9 +562,15 @@ class VllmConfig:
|
||||
if model_config.runner_type != "generate":
|
||||
return False
|
||||
|
||||
if getattr(model_config, "is_hybrid", False):
|
||||
return False
|
||||
|
||||
if getattr(model_config, "is_attention_free", False):
|
||||
return False
|
||||
architectures = getattr(model_config, "architectures", [])
|
||||
return any(
|
||||
arch in DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES for arch in architectures
|
||||
return (
|
||||
any(arch in DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES for arch in architectures)
|
||||
or not model_config.is_moe
|
||||
)
|
||||
|
||||
@property
|
||||
|
||||
Reference in New Issue
Block a user