[DCP] Support Decode Context Parallel (DCP) for GQA with Flashinfer (#25438)
Signed-off-by: gaojc <1055866782@qq.com> Signed-off-by: Jingchun Gao <gaojingchun1@huawei.com> Signed-off-by: Jingchun Gao <63247409+gjc0824@users.noreply.github.com> Signed-off-by: QiuChunshuo <qiuchunshuo@huawei.com> Co-authored-by: gaojingchun (A) <g00955623@china.huawei.com> Co-authored-by: Jingchun Gao <gaojingchun1@huawei.com> Co-authored-by: QiuChunshuo <qiuchunshuo@huawei.com>
This commit is contained in:
co-authored by
gaojingchun
Jingchun Gao
QiuChunshuo
parent
41b92f7d38
commit
4516d44b7f
@@ -259,6 +259,7 @@ def use_trtllm_attention(
|
||||
num_kv_heads: int,
|
||||
num_tokens: int,
|
||||
max_seq_len: int,
|
||||
dcp_world_size: int,
|
||||
kv_cache_dtype: str,
|
||||
q_dtype: torch.dtype,
|
||||
is_prefill: bool,
|
||||
@@ -272,6 +273,14 @@ def use_trtllm_attention(
|
||||
if force_use_trtllm is not None and not force_use_trtllm:
|
||||
return False
|
||||
|
||||
# Decode context parallel is not supported
|
||||
if dcp_world_size > 1:
|
||||
logger.warning_once(
|
||||
"Trtllm does not support returning LSE and as a result "
|
||||
"does not support DCP, reverting to FlashInfer"
|
||||
)
|
||||
return False
|
||||
|
||||
# The platform is not supported
|
||||
if not supports_trtllm_attention():
|
||||
if force_use_trtllm:
|
||||
|
||||
Reference in New Issue
Block a user