[Kernel] unified_attention for Attention.forward (#11967)

Signed-off-by: Chen Zhang <zhangch99@outlook.com>
This commit is contained in:
Chen Zhang
2025-01-13 19:28:53 +08:00
committed by GitHub
parent 5340a30d01
commit 0f8cafe2d1
10 changed files with 87 additions and 45 deletions
-1
View File
@@ -2171,5 +2171,4 @@ def bind_kv_cache(
forward_ctx = ctx[layer_name]
assert len(forward_ctx.kv_cache) == len(kv_cache)
for ve, ve_kv_cache in enumerate(kv_cache):
assert forward_ctx.kv_cache[ve].numel() == 0
forward_ctx.kv_cache[ve] = ve_kv_cache[kv_cache_idx]