[Kernel][UX] Add --linear-backend arg for linear kernel selection (#39538)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -90,9 +90,9 @@ def test_models(example_prompts, model_name) -> None:
|
||||
EAGER = [True, False]
|
||||
|
||||
SM_100_NVFP4_BACKENDS = [
|
||||
"flashinfer-cudnn",
|
||||
"flashinfer-trtllm",
|
||||
"flashinfer-cutlass",
|
||||
"flashinfer_cudnn",
|
||||
"flashinfer_trtllm",
|
||||
"flashinfer_cutlass",
|
||||
]
|
||||
|
||||
|
||||
@@ -102,12 +102,12 @@ SM_100_NVFP4_BACKENDS = [
|
||||
"backend",
|
||||
[
|
||||
"emulation",
|
||||
"flashinfer-cudnn",
|
||||
"flashinfer-trtllm", # the small seq_len ensures trtllm_8x4_layout backend is used
|
||||
"flashinfer-cutlass",
|
||||
"flashinfer_cudnn",
|
||||
"flashinfer_trtllm", # the small seq_len ensures trtllm_8x4_layout backend is used
|
||||
"flashinfer_cutlass",
|
||||
],
|
||||
)
|
||||
def test_nvfp4(vllm_runner, model, eager, backend, monkeypatch):
|
||||
def test_nvfp4(vllm_runner, model, eager, backend):
|
||||
if (
|
||||
not current_platform.has_device_capability(100)
|
||||
and backend in SM_100_NVFP4_BACKENDS
|
||||
@@ -116,8 +116,7 @@ def test_nvfp4(vllm_runner, model, eager, backend, monkeypatch):
|
||||
f"The backend {backend} is not supported with current_platform.has_device_capability(100) == False"
|
||||
)
|
||||
|
||||
monkeypatch.setenv("VLLM_NVFP4_GEMM_BACKEND", backend)
|
||||
with vllm_runner(model, enforce_eager=eager) as llm:
|
||||
with vllm_runner(model, enforce_eager=eager, linear_backend=backend) as llm:
|
||||
output = llm.generate_greedy(["1 2 3 4 5"], max_tokens=2)
|
||||
assert output[0][1] == "1 2 3 4 5 6"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user