[CPU][Spec Decode] Support DFlash speculative decoding for GDN models on CPU (#46090)
Signed-off-by: guybd <guy.boudoukh@intel.com> Signed-off-by: Guy Boudoukh <guy.boudoukh@intel.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
co-authored by
Cursor
Li, Jiang <jiang1.li@intel.com>
parent
d973cce3ca
commit
05fa8183a6
@@ -3252,6 +3252,40 @@ def fused_sigmoid_gating_delta_rule_update_cpu(
|
||||
)
|
||||
|
||||
|
||||
def fused_sigmoid_gating_delta_rule_update_spec_cpu(
|
||||
A_log: torch.Tensor,
|
||||
dt_bias: torch.Tensor,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
a: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
initial_state_source: torch.Tensor,
|
||||
spec_state_indices: torch.Tensor,
|
||||
num_accepted_tokens: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
use_qk_l2norm_in_kernel: bool,
|
||||
softplus_beta: float = 1.0,
|
||||
softplus_threshold: float = 20.0,
|
||||
) -> torch.Tensor:
|
||||
return torch.ops._C.fused_sigmoid_gating_delta_rule_update_spec_cpu(
|
||||
A_log,
|
||||
dt_bias,
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
a,
|
||||
b,
|
||||
initial_state_source,
|
||||
spec_state_indices,
|
||||
num_accepted_tokens,
|
||||
cu_seqlens,
|
||||
use_qk_l2norm_in_kernel,
|
||||
softplus_beta,
|
||||
softplus_threshold,
|
||||
)
|
||||
|
||||
|
||||
def fused_gdn_gating_cpu(
|
||||
A_log: torch.Tensor,
|
||||
a: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user