[kernel] Fused embedding kernel (#53677)

Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
Jee Jee Li
2026-08-31 11:34:14 +00:00
committed by GitHub
parent bed3280f50
commit e2c8eeac40
8 changed files with 580 additions and 10 deletions
+9
View File
@@ -700,6 +700,14 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
// LongCat n-gram embedding index kernel. All tensor args are marked mutable
// to match the (non-const) stable-Tensor& C++ signature; only ne_token_table
// and n_gram_ids are actually written in place.
// Fused vocab-parallel embedding: gather the rows this rank owns and write
// zeros for the rest, so the following all-reduce reconstructs the full
// embedding.
ops.def(
"vocab_parallel_embedding(Tensor! out, Tensor input_ids, Tensor weight, "
"int org_vocab_start_index, int org_vocab_end_index, "
"int num_org_vocab_padding, int added_vocab_start_index, "
"int added_vocab_end_index) -> ()");
ops.def(
"ngram_compute_n_gram_ids(int ne_n, int ne_k, Tensor(a!) ne_weights, "
"Tensor(b!) ne_mods, Tensor(c!) exclusive_ne_embedder_size_sums, "
@@ -711,6 +719,7 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
// LongCat n-gram embedding index kernel.
ops.impl("ngram_compute_n_gram_ids", TORCH_BOX(&ngram_compute_n_gram_ids));
ops.impl("vocab_parallel_embedding", TORCH_BOX(&vocab_parallel_embedding));
// Per-token group quantization
ops.impl("per_token_group_fp8_quant", TORCH_BOX(&per_token_group_quant_fp8));