[Bugfix] Fix Qwen4Exp FP8 PLE pinned lookup kernel failing to compile below SM89 (#60142)
Signed-off-by: Vadim Gimpelson <vadim.gimpelson@gmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
co-authored by
Claude
Thien Tran
parent
9465e1c783
commit
b0b23a48d4
@@ -373,29 +373,33 @@ class Qwen4ExpPLEDeviceEmbedding(Qwen4ExpPLEEmbedding):
|
||||
|
||||
@triton.jit
|
||||
def _lookup_ple_embedding_from_pinned_kernel(
|
||||
weight_ptr,
|
||||
weight_ptr: tl.pointer_type(tl.uint8), # type: ignore[valid-type]
|
||||
ids_ptr,
|
||||
output_ptr,
|
||||
embedding_dim,
|
||||
output_ptr: tl.pointer_type(tl.uint8), # type: ignore[valid-type]
|
||||
row_bytes,
|
||||
tp_vocab_start,
|
||||
tp_vocab_end,
|
||||
BLOCK_D: tl.constexpr,
|
||||
):
|
||||
"""Look up TP-owned PLE rows through a CUDA view of pinned host memory."""
|
||||
row_id = tl.program_id(0)
|
||||
"""Copy TP-owned PLE rows as raw bytes from a CUDA view of pinned host memory.
|
||||
|
||||
Byte pointers keep the storage dtype out of the kernel signature, so
|
||||
dtypes Triton cannot lower on the GPU (FP8 E4M3FN before SM89) still work.
|
||||
"""
|
||||
row_id = tl.program_id(0).to(tl.int64)
|
||||
global_idx = tl.load(ids_ptr + row_id)
|
||||
in_range = (global_idx >= tp_vocab_start) & (global_idx < tp_vocab_end)
|
||||
local_idx = tl.where(in_range, global_idx - tp_vocab_start, 0)
|
||||
offsets = tl.arange(0, BLOCK_D)
|
||||
store_mask = offsets < embedding_dim
|
||||
store_mask = offsets < row_bytes
|
||||
load_mask = store_mask & in_range
|
||||
values = tl.load(
|
||||
weight_ptr + local_idx * embedding_dim + offsets,
|
||||
weight_ptr + local_idx * row_bytes + offsets,
|
||||
mask=load_mask,
|
||||
other=0.0,
|
||||
other=0,
|
||||
)
|
||||
tl.store(
|
||||
output_ptr + row_id * embedding_dim + offsets,
|
||||
output_ptr + row_id * row_bytes + offsets,
|
||||
values,
|
||||
mask=store_mask,
|
||||
)
|
||||
@@ -433,7 +437,8 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
)
|
||||
self._uva_weight = get_accelerator_view_from_cpu_tensor(self.weight)
|
||||
self._block_d = triton.next_power_of_2(self.embedding_dim)
|
||||
self._row_bytes = self.embedding_dim * self.weight.element_size()
|
||||
self._block_d = triton.next_power_of_2(self._row_bytes)
|
||||
self._prefetch_stream: torch.cuda.Stream | None = None
|
||||
self._prefetch_buffer: torch.Tensor | None = None
|
||||
self._prefetch_alloc_lock = threading.Lock()
|
||||
@@ -485,7 +490,7 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
|
||||
self._uva_weight,
|
||||
flat_ids,
|
||||
output,
|
||||
self.embedding_dim,
|
||||
self._row_bytes,
|
||||
self.shard_indices.org_vocab_start_index,
|
||||
self.shard_indices.org_vocab_end_index,
|
||||
BLOCK_D=self._block_d,
|
||||
|
||||
@@ -128,6 +128,8 @@ class TritonLanguagePlaceholder(types.ModuleType):
|
||||
self.dtype = None
|
||||
self.int64 = None
|
||||
self.int32 = None
|
||||
self.uint8 = None
|
||||
self.pointer_type = lambda element_ty: None
|
||||
self.tensor = None
|
||||
self.exp = None
|
||||
self.exp2 = None
|
||||
|
||||
Reference in New Issue
Block a user