[MRV2] Support stock torch.compile mode (#59079)
Signed-off-by: cheesecake <farzanaman99@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Nick Hill
Wentao Ye
parent
4406874e7c
commit
4e59d53ab5
@@ -215,7 +215,9 @@ def test_use_cudagraphs(
|
||||
|
||||
# forked needed to workaround https://github.com/vllm-project/vllm/issues/21073
|
||||
@pytest.mark.forked
|
||||
def test_stock_torch_compile(vllm_runner, monkeypatch):
|
||||
@pytest.mark.parametrize("use_v2_model_runner", [False, True])
|
||||
def test_stock_torch_compile(vllm_runner, monkeypatch, use_v2_model_runner):
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", str(int(use_v2_model_runner)))
|
||||
# Disable multiprocessing so that the counter is in the same process
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
|
||||
@@ -226,9 +228,10 @@ def test_stock_torch_compile(vllm_runner, monkeypatch):
|
||||
"facebook/opt-125m",
|
||||
compilation_config={"mode": CompilationMode.STOCK_TORCH_COMPILE},
|
||||
gpu_memory_utilization=0.4,
|
||||
) as _,
|
||||
) as runner,
|
||||
):
|
||||
pass
|
||||
outputs = runner.generate_greedy(["Hello, my name is"], max_tokens=5)
|
||||
assert outputs[0][0]
|
||||
|
||||
|
||||
# forked needed to workaround https://github.com/vllm-project/vllm/issues/21073
|
||||
|
||||
Reference in New Issue
Block a user