[Bugfix] Fix Qwen4Exp FP8 PLE pinned lookup kernel failing to compile below SM89 (#60142)

Signed-off-by: Vadim Gimpelson <vadim.gimpelson@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
Vadim Gimpelson
2026-10-06 01:56:44 +00:00
committed by GitHub
co-authored by Claude Thien Tran
parent 9465e1c783
commit b0b23a48d4
2 changed files with 18 additions and 11 deletions
+16 -11
View File
@@ -373,29 +373,33 @@ class Qwen4ExpPLEDeviceEmbedding(Qwen4ExpPLEEmbedding):
@triton.jit
def _lookup_ple_embedding_from_pinned_kernel(
weight_ptr,
weight_ptr: tl.pointer_type(tl.uint8), # type: ignore[valid-type]
ids_ptr,
output_ptr,
embedding_dim,
output_ptr: tl.pointer_type(tl.uint8), # type: ignore[valid-type]
row_bytes,
tp_vocab_start,
tp_vocab_end,
BLOCK_D: tl.constexpr,
):
"""Look up TP-owned PLE rows through a CUDA view of pinned host memory."""
row_id = tl.program_id(0)
"""Copy TP-owned PLE rows as raw bytes from a CUDA view of pinned host memory.
Byte pointers keep the storage dtype out of the kernel signature, so
dtypes Triton cannot lower on the GPU (FP8 E4M3FN before SM89) still work.
"""
row_id = tl.program_id(0).to(tl.int64)
global_idx = tl.load(ids_ptr + row_id)
in_range = (global_idx >= tp_vocab_start) & (global_idx < tp_vocab_end)
local_idx = tl.where(in_range, global_idx - tp_vocab_start, 0)
offsets = tl.arange(0, BLOCK_D)
store_mask = offsets < embedding_dim
store_mask = offsets < row_bytes
load_mask = store_mask & in_range
values = tl.load(
weight_ptr + local_idx * embedding_dim + offsets,
weight_ptr + local_idx * row_bytes + offsets,
mask=load_mask,
other=0.0,
other=0,
)
tl.store(
output_ptr + row_id * embedding_dim + offsets,
output_ptr + row_id * row_bytes + offsets,
values,
mask=store_mask,
)
@@ -433,7 +437,8 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
data_parallel_rank=data_parallel_rank,
)
self._uva_weight = get_accelerator_view_from_cpu_tensor(self.weight)
self._block_d = triton.next_power_of_2(self.embedding_dim)
self._row_bytes = self.embedding_dim * self.weight.element_size()
self._block_d = triton.next_power_of_2(self._row_bytes)
self._prefetch_stream: torch.cuda.Stream | None = None
self._prefetch_buffer: torch.Tensor | None = None
self._prefetch_alloc_lock = threading.Lock()
@@ -485,7 +490,7 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
self._uva_weight,
flat_ids,
output,
self.embedding_dim,
self._row_bytes,
self.shard_indices.org_vocab_start_index,
self.shard_indices.org_vocab_end_index,
BLOCK_D=self._block_d,
+2
View File
@@ -128,6 +128,8 @@ class TritonLanguagePlaceholder(types.ModuleType):
self.dtype = None
self.int64 = None
self.int32 = None
self.uint8 = None
self.pointer_type = lambda element_ty: None
self.tensor = None
self.exp = None
self.exp2 = None