[Feature] Add process_weights_after_loading to AttentionImpl (#26870)

Signed-off-by: rongfu.leng <rongfu.leng@daocloud.io>
This commit is contained in:
rongfu.leng
2025-10-16 08:02:30 -07:00
committed by GitHub
parent 43721bc67f
commit 5afd3276df
3 changed files with 9 additions and 10 deletions
+5
View File
@@ -833,6 +833,11 @@ class FlashInferImpl(AttentionImpl):
return self.support_trtllm_attn
# FlashInfer requires attention sinks to be float32
def process_weights_after_loading(self, act_dtype: torch.dtype):
if self.sinks is not None and self.sinks.dtype != torch.float32:
self.sinks = self.sinks.to(torch.float32)
def forward(
self,
layer: torch.nn.Module,