diff --git a/vllm/models/qwen4_exp/common/ngram_embedding.py b/vllm/models/qwen4_exp/common/ngram_embedding.py index 24eac4f876..6dad6b7b5a 100644 --- a/vllm/models/qwen4_exp/common/ngram_embedding.py +++ b/vllm/models/qwen4_exp/common/ngram_embedding.py @@ -373,29 +373,33 @@ class Qwen4ExpPLEDeviceEmbedding(Qwen4ExpPLEEmbedding): @triton.jit def _lookup_ple_embedding_from_pinned_kernel( - weight_ptr, + weight_ptr: tl.pointer_type(tl.uint8), # type: ignore[valid-type] ids_ptr, - output_ptr, - embedding_dim, + output_ptr: tl.pointer_type(tl.uint8), # type: ignore[valid-type] + row_bytes, tp_vocab_start, tp_vocab_end, BLOCK_D: tl.constexpr, ): - """Look up TP-owned PLE rows through a CUDA view of pinned host memory.""" - row_id = tl.program_id(0) + """Copy TP-owned PLE rows as raw bytes from a CUDA view of pinned host memory. + + Byte pointers keep the storage dtype out of the kernel signature, so + dtypes Triton cannot lower on the GPU (FP8 E4M3FN before SM89) still work. + """ + row_id = tl.program_id(0).to(tl.int64) global_idx = tl.load(ids_ptr + row_id) in_range = (global_idx >= tp_vocab_start) & (global_idx < tp_vocab_end) local_idx = tl.where(in_range, global_idx - tp_vocab_start, 0) offsets = tl.arange(0, BLOCK_D) - store_mask = offsets < embedding_dim + store_mask = offsets < row_bytes load_mask = store_mask & in_range values = tl.load( - weight_ptr + local_idx * embedding_dim + offsets, + weight_ptr + local_idx * row_bytes + offsets, mask=load_mask, - other=0.0, + other=0, ) tl.store( - output_ptr + row_id * embedding_dim + offsets, + output_ptr + row_id * row_bytes + offsets, values, mask=store_mask, ) @@ -433,7 +437,8 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding): data_parallel_rank=data_parallel_rank, ) self._uva_weight = get_accelerator_view_from_cpu_tensor(self.weight) - self._block_d = triton.next_power_of_2(self.embedding_dim) + self._row_bytes = self.embedding_dim * self.weight.element_size() + self._block_d = triton.next_power_of_2(self._row_bytes) self._prefetch_stream: torch.cuda.Stream | None = None self._prefetch_buffer: torch.Tensor | None = None self._prefetch_alloc_lock = threading.Lock() @@ -485,7 +490,7 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding): self._uva_weight, flat_ids, output, - self.embedding_dim, + self._row_bytes, self.shard_indices.org_vocab_start_index, self.shard_indices.org_vocab_end_index, BLOCK_D=self._block_d, diff --git a/vllm/triton_utils/importing.py b/vllm/triton_utils/importing.py index e312564c0b..12f19cc52b 100644 --- a/vllm/triton_utils/importing.py +++ b/vllm/triton_utils/importing.py @@ -128,6 +128,8 @@ class TritonLanguagePlaceholder(types.ModuleType): self.dtype = None self.int64 = None self.int32 = None + self.uint8 = None + self.pointer_type = lambda element_ty: None self.tensor = None self.exp = None self.exp2 = None