[PERF] Remove TRTLLM Gen attn kernel limitation max_seq_len <=131072 (#28755)
Signed-off-by: Vadim Gimpelson <vadim.gimpelson@gmail.com>
This commit is contained in:
@@ -319,14 +319,12 @@ def use_trtllm_attention(
|
||||
# Environment variable not set - use auto-detection
|
||||
if is_prefill:
|
||||
# Prefill auto-detection
|
||||
use_trtllm = max_seq_len <= 131072 and kv_cache_dtype == "auto"
|
||||
use_trtllm = kv_cache_dtype == "auto"
|
||||
if use_trtllm:
|
||||
logger.warning_once("Using TRTLLM prefill attention (auto-detected).")
|
||||
else:
|
||||
# Decode auto-detection
|
||||
use_trtllm = (
|
||||
num_tokens <= 256 and max_seq_len <= 131072 and kv_cache_dtype == "auto"
|
||||
)
|
||||
use_trtllm = num_tokens <= 256 and kv_cache_dtype == "auto"
|
||||
if use_trtllm:
|
||||
logger.warning_once("Using TRTLLM decode attention (auto-detected).")
|
||||
return use_trtllm
|
||||
|
||||
Reference in New Issue
Block a user