DCP supports hybrid attention (#40996)
Signed-off-by: YanXu <yancey.yx@alibaba-inc.com> Signed-off-by: Jingyi Yang <girasoleyang@gmail.com> Co-authored-by: Jingyi Yang <girasoleyang@gmail.com>
This commit is contained in:
@@ -1881,11 +1881,8 @@ class ModelConfig:
|
||||
else:
|
||||
# for generative models
|
||||
if attn_type == "hybrid":
|
||||
logger.debug(
|
||||
"Hybrid models do not support prefix caching since the feature "
|
||||
"is still experimental."
|
||||
)
|
||||
return False
|
||||
logger.debug("Generative hybrid models support prefix caching.")
|
||||
return True
|
||||
elif attn_type == "attention_free":
|
||||
logger.debug(
|
||||
"Attention free models do not support prefix caching since the "
|
||||
|
||||
Reference in New Issue
Block a user