[CPU][Spec Decode] Support DFlash speculative decoding for GDN models on CPU (#46090)
Signed-off-by: guybd <guy.boudoukh@intel.com> Signed-off-by: Guy Boudoukh <guy.boudoukh@intel.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
co-authored by
Cursor
Li, Jiang <jiang1.li@intel.com>
parent
d973cce3ca
commit
05fa8183a6
@@ -120,6 +120,14 @@ at::Tensor fused_sigmoid_gating_delta_rule_update_cpu(
|
||||
bool use_qk_l2norm_in_kernel, double softplus_beta = 1.0,
|
||||
double softplus_threshold = 20.0);
|
||||
|
||||
at::Tensor fused_sigmoid_gating_delta_rule_update_spec_cpu(
|
||||
const at::Tensor& A_log, const at::Tensor& dt_bias, const at::Tensor& q,
|
||||
const at::Tensor& k, const at::Tensor& v, const at::Tensor& a,
|
||||
const at::Tensor& b, at::Tensor& initial_state_source,
|
||||
const at::Tensor& spec_state_indices, const at::Tensor& num_accepted_tokens,
|
||||
const at::Tensor& cu_seqlens, bool use_qk_l2norm_in_kernel,
|
||||
double softplus_beta = 1.0, double softplus_threshold = 20.0);
|
||||
|
||||
std::tuple<at::Tensor, at::Tensor> fused_gdn_gating_cpu(
|
||||
const at::Tensor& A_log, const at::Tensor& a, const at::Tensor& b,
|
||||
const at::Tensor& dt_bias);
|
||||
@@ -508,6 +516,15 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
|
||||
"softplus_threshold=20.0) -> Tensor");
|
||||
ops.impl("fused_sigmoid_gating_delta_rule_update_cpu", torch::kCPU,
|
||||
&fused_sigmoid_gating_delta_rule_update_cpu);
|
||||
ops.def(
|
||||
"fused_sigmoid_gating_delta_rule_update_spec_cpu(Tensor A_log, Tensor "
|
||||
"dt_bias, Tensor q, Tensor k, Tensor v, Tensor a, Tensor b, "
|
||||
"Tensor(a!) initial_state_source, Tensor spec_state_indices, "
|
||||
"Tensor num_accepted_tokens, Tensor cu_seqlens, bool "
|
||||
"use_qk_l2norm_in_kernel, float softplus_beta=1.0, float "
|
||||
"softplus_threshold=20.0) -> Tensor");
|
||||
ops.impl("fused_sigmoid_gating_delta_rule_update_spec_cpu", torch::kCPU,
|
||||
&fused_sigmoid_gating_delta_rule_update_spec_cpu);
|
||||
ops.def(
|
||||
"fused_gdn_gating_cpu(Tensor A_log, Tensor a, Tensor b, Tensor dt_bias) "
|
||||
"-> (Tensor, Tensor)");
|
||||
|
||||
Reference in New Issue
Block a user