register custom op for flash attn and use from torch.ops (#7536)

This commit is contained in:
youkaichao
2024-08-15 22:38:56 -07:00
committed by GitHub
parent 50b8d08dbd
commit 54bd9a03c4
5 changed files with 221 additions and 42 deletions
+20
View File
@@ -0,0 +1,20 @@
import os
import pytest
@pytest.mark.parametrize("model", ["meta-llama/Meta-Llama-3-8B"])
def test_full_graph(model):
# make sure these models can be captured in full graph mode
os.environ["VLLM_TEST_DYNAMO_GRAPH_CAPTURE"] = "1"
from vllm import LLM, SamplingParams
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
]
sampling_params = SamplingParams(temperature=0)
llm = LLM(model="meta-llama/Meta-Llama-3-8B")
llm.generate(prompts, sampling_params)