[Core] MRV2 support for fast-prefill (#56145)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -52,12 +52,23 @@ use_fork_for_test = (
|
||||
|
||||
|
||||
@use_fork_for_test
|
||||
@pytest.mark.parametrize("kv_sharing_fast_prefill", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"kv_sharing_fast_prefill,use_v2_model_runner",
|
||||
[
|
||||
(False, False),
|
||||
(True, False),
|
||||
(True, True),
|
||||
# (False, True) omitted: fast-prefill-off behavior is runner-generic,
|
||||
# and skipping it saves ~9 min of mostly torch.compile time in CI. The
|
||||
# no-silent-fallback assertion below is exercised by the V2 cases.
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("enforce_eager", [True, False])
|
||||
def test_kv_sharing_fast_prefill(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
kv_sharing_fast_prefill: bool,
|
||||
enforce_eager: bool,
|
||||
use_v2_model_runner: bool,
|
||||
):
|
||||
if not enforce_eager and current_platform.is_rocm():
|
||||
# Relevant context: https://github.com/vllm-project/vllm/pull/29244
|
||||
@@ -84,6 +95,7 @@ def test_kv_sharing_fast_prefill(
|
||||
m.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
||||
else:
|
||||
m.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
m.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_v2_model_runner else "0")
|
||||
|
||||
prompts, answer, indices = prep_prompts(batch_size)
|
||||
|
||||
@@ -95,6 +107,8 @@ def test_kv_sharing_fast_prefill(
|
||||
kv_sharing_fast_prefill=kv_sharing_fast_prefill,
|
||||
attention_backend="TRITON_ATTN",
|
||||
)
|
||||
# Guard against a silent fallback to the other model runner.
|
||||
assert llm.llm_engine.vllm_config.use_v2_model_runner == use_v2_model_runner
|
||||
responses = llm.generate(prompts, sampling_params)
|
||||
check_answers(
|
||||
indices,
|
||||
|
||||
Reference in New Issue
Block a user