[KERNEL] Sampler. CUDA kernel for applying repetition penalty (#18437)

This commit is contained in:
Vadim Gimpelson
2025-06-03 21:13:01 -07:00
committed by GitHub
parent 1409ef9134
commit 5d6d1adf15
7 changed files with 218 additions and 9 deletions
+7
View File
@@ -170,6 +170,13 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
"float epsilon) -> ()");
ops.impl("fused_add_rms_norm", torch::kCUDA, &fused_add_rms_norm);
// Apply repetition penalties to logits in-place
ops.def(
"apply_repetition_penalties_(Tensor! logits, Tensor prompt_mask, "
"Tensor output_mask, Tensor repetition_penalties) -> ()");
ops.impl("apply_repetition_penalties_", torch::kCUDA,
&apply_repetition_penalties_);
// Layernorm-quant
// Apply Root Mean Square (RMS) Normalization to the input tensor.
ops.def(