[CPU][Perf] Add vectorized Sampler Kernel (#53913)

Signed-off-by: Rehan Khan <Rehan.Khan7@ibm.com>
Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
Rehan Khan
2026-09-30 15:16:28 +08:00
committed by GitHub
co-authored by Li, Jiang <jiang1.li@intel.com>
parent 77e8b02c8d
commit 583637f126
7 changed files with 528 additions and 11 deletions
+12
View File
@@ -393,6 +393,11 @@ void mamba_chunk_scan_fwd_cpu_impl(at::Tensor& out, at::Tensor& final_states,
const c10::optional<at::Tensor>& z,
const at::Tensor& cu_seqlens);
torch::Tensor fused_gumbel_argmax(const torch::Tensor& logits,
const torch::Tensor& seeds);
torch::Tensor greedy_argmax(const torch::Tensor& logits);
void init_cpu_memory_env(std::vector<int64_t> node_ids);
namespace cpu_utils {
@@ -969,6 +974,13 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
ops.def("init_cpu_memory_env(SymInt[] node_ids) -> ()", &init_cpu_memory_env);
// Fused sampling kernels
ops.def("fused_gumbel_argmax(Tensor logits, Tensor seeds) -> Tensor");
ops.impl("fused_gumbel_argmax", torch::kCPU, &fused_gumbel_argmax);
ops.def("greedy_argmax(Tensor logits) -> Tensor");
ops.impl("greedy_argmax", torch::kCPU, &greedy_argmax);
// Speculative decoding kernels
ops.def(
"eagle_prepare_inputs_padded_kernel_impl(Tensor cu_num_draft_tokens, "