[Bugfix] Refactor Flashinfer TRTLLM attention kernel selection logic (#24600)
Signed-off-by: elvischenv <219235043+elvischenv@users.noreply.github.com> Co-authored-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Michael Goin
parent
9f882d8791
commit
e67a79db03
+5
-2
@@ -1223,9 +1223,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_USE_CUDNN_PREFILL":
|
||||
lambda: bool(int(os.getenv("VLLM_USE_CUDNN_PREFILL", "0"))),
|
||||
|
||||
# If set to 1, use the TRTLLM attention backend in flashinfer.
|
||||
# If set to 1/True, use the TRTLLM attention backend in flashinfer.
|
||||
# If set to 0/False, use the default attention backend in flashinfer.
|
||||
# If not set, auto-detect the attention backend in flashinfer.
|
||||
"VLLM_USE_TRTLLM_ATTENTION":
|
||||
lambda: os.getenv("VLLM_USE_TRTLLM_ATTENTION", None),
|
||||
lambda: (None if "VLLM_USE_TRTLLM_ATTENTION" not in os.environ else
|
||||
os.environ["VLLM_USE_TRTLLM_ATTENTION"].lower() in ("1", "true")),
|
||||
|
||||
# If set to 1, when we use fp8 kv, we do not quantize Q to fp8
|
||||
"VLLM_FLASHINFER_DISABLE_Q_QUANTIZATION":
|
||||
|
||||
Reference in New Issue
Block a user