[kernel] Fused embedding kernel (#53677)

Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
Jee Jee Li
2026-08-31 11:34:14 +00:00
committed by GitHub
parent bed3280f50
commit e2c8eeac40
8 changed files with 580 additions and 10 deletions
@@ -9,6 +9,7 @@ import torch.nn.functional as F
from torch.nn.parameter import Parameter
import vllm.envs as envs
from vllm import _custom_ops as ops
from vllm.distributed import (
divide,
get_tensor_model_parallel_rank,
@@ -325,6 +326,14 @@ class VocabParallelEmbedding(PluggableLayer):
- self.shard_indices.added_vocab_start_index
)
# The fused kernel masks, shifts and gathers in one pass; it assumes a
# plain row-major shard, so it only applies to unquantized weights.
self.use_fused_embedding = (
self.tp_size > 1
and current_platform.is_cuda()
and isinstance(quant_method, UnquantizedEmbeddingMethod)
)
self.quant_method.create_weights(
self,
self.embedding_dim,
@@ -488,8 +497,24 @@ class VocabParallelEmbedding(PluggableLayer):
param[loaded_weight.shape[0] :].data.fill_(0)
def forward(self, input_):
if self.tp_size > 1:
# Build the mask.
if self.tp_size == 1:
return self.quant_method.embedding(self, input_.long())
if self.use_fused_embedding:
output_parallel = ops.vocab_parallel_embedding(
input_ if input_.ndim == 1 else input_.reshape(-1),
self.weight,
self.shard_indices.org_vocab_start_index,
self.shard_indices.org_vocab_end_index,
self.shard_indices.num_org_vocab_padding,
self.shard_indices.added_vocab_start_index,
self.shard_indices.added_vocab_end_index,
)
if input_.ndim != 1:
output_parallel = output_parallel.view(
*input_.shape, self.embedding_dim
)
else:
masked_input, input_mask = get_masked_input_and_mask(
input_,
self.shard_indices.org_vocab_start_index,
@@ -498,12 +523,7 @@ class VocabParallelEmbedding(PluggableLayer):
self.shard_indices.added_vocab_start_index,
self.shard_indices.added_vocab_end_index,
)
else:
masked_input = input_
# Get the embeddings.
output_parallel = self.quant_method.embedding(self, masked_input.long())
# Mask the output embedding.
if self.tp_size > 1:
output_parallel = self.quant_method.embedding(self, masked_input.long())
if output_parallel.dtype in (
torch.float8_e4m3fn,
torch.float8_e5m2,
@@ -514,8 +534,8 @@ class VocabParallelEmbedding(PluggableLayer):
output = tensor_model_parallel_all_reduce(comm_output)
return output.view(output_parallel.dtype)
output_parallel.masked_fill_(input_mask.unsqueeze(-1), 0)
return tensor_model_parallel_all_reduce(output_parallel)
return output_parallel
# Reduce across all the model parallel GPUs.
return tensor_model_parallel_all_reduce(output_parallel)
def extra_repr(self) -> str:
s = f"num_embeddings={self.num_embeddings}"