[CPU][Spec Decode] Support DFlash speculative decoding for GDN models on CPU (#46090)
Signed-off-by: guybd <guy.boudoukh@intel.com> Signed-off-by: Guy Boudoukh <guy.boudoukh@intel.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
co-authored by
Cursor
Li, Jiang <jiang1.li@intel.com>
parent
d973cce3ca
commit
05fa8183a6
@@ -239,12 +239,18 @@ def dispatch_cpu_unquantized_gemm(
|
||||
# For now it should be a causal_conv1d op
|
||||
if torch.cpu._is_amx_tile_supported():
|
||||
# prepack conv weight
|
||||
layer.weight.data = ops.causal_conv1d_weight_pack(
|
||||
unpacked = (
|
||||
layer.weight.view(
|
||||
layer.weight.size(0),
|
||||
layer.weight.size(2),
|
||||
)
|
||||
.contiguous()
|
||||
.clone()
|
||||
)
|
||||
# Stash the un-packed (dim, width) weight so the speculative-decode
|
||||
# GDN path (which uses torch conv, not the AMX kernel) can use it.
|
||||
layer._cpu_unpacked_conv_weight = unpacked
|
||||
layer.weight.data = ops.causal_conv1d_weight_pack(unpacked)
|
||||
return
|
||||
|
||||
N, K = layer.weight.size()
|
||||
|
||||
Reference in New Issue
Block a user