This commit is contained in:
@@ -41,7 +41,6 @@ if current_platform.is_cuda():
|
||||
def _dsv4_topk_kernel(
|
||||
gating_output_ptr,
|
||||
correction_bias_ptr,
|
||||
is_padding_ptr,
|
||||
topk_weights_ptr,
|
||||
topk_ids_ptr,
|
||||
routed_scaling_factor,
|
||||
@@ -50,7 +49,6 @@ if current_platform.is_cuda():
|
||||
image_sentinel_lo,
|
||||
NUM_EXPERTS: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
HAS_PADDING: tl.constexpr,
|
||||
HAS_VL: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
@@ -108,11 +106,6 @@ if current_platform.is_cuda():
|
||||
output_mask = topk_offsets < 6
|
||||
output_offsets = row * 6 + topk_offsets
|
||||
|
||||
if HAS_PADDING:
|
||||
is_padding = tl.load(is_padding_ptr + row)
|
||||
selected_weights = tl.where(is_padding, 0.0, selected_weights)
|
||||
selected_ids = tl.where(is_padding, -1, selected_ids)
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
@@ -125,20 +118,11 @@ def dsv4_topk(
|
||||
correction_bias: torch.Tensor,
|
||||
indices_dtype: torch.dtype,
|
||||
routed_scaling_factor: float,
|
||||
is_padding: torch.Tensor | None = None,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
bias_vl: torch.Tensor | None = None,
|
||||
image_sentinel_lo: int = 0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
num_tokens, num_experts = gating_output.shape
|
||||
if is_padding is not None:
|
||||
assert is_padding.dtype == torch.bool
|
||||
assert is_padding.shape == (num_tokens,)
|
||||
assert is_padding.device == gating_output.device
|
||||
assert is_padding.is_contiguous()
|
||||
assert indices_dtype in (torch.int32, torch.int64), (
|
||||
"Padding requires a signed indices dtype for the -1 sentinel."
|
||||
)
|
||||
has_vl = bias_vl is not None and image_sentinel_lo > 0
|
||||
if bias_vl is not None:
|
||||
assert input_ids is not None, "bias_vl routing requires input_ids"
|
||||
@@ -152,7 +136,6 @@ def dsv4_topk(
|
||||
_dsv4_topk_kernel[(num_tokens,)](
|
||||
gating_output,
|
||||
correction_bias,
|
||||
is_padding,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
routed_scaling_factor,
|
||||
@@ -161,7 +144,6 @@ def dsv4_topk(
|
||||
image_sentinel_lo,
|
||||
NUM_EXPERTS=num_experts,
|
||||
BLOCK_N=triton.next_power_of_2(num_experts),
|
||||
HAS_PADDING=is_padding is not None,
|
||||
HAS_VL=has_vl,
|
||||
num_warps=1,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
|
||||
@@ -148,13 +148,9 @@ def fused_topk_bias(
|
||||
)
|
||||
|
||||
output_indices_dtype = torch.int32 if indices_type is None else indices_type
|
||||
padding_mask = _get_padding_mask(gating_output.shape[0])
|
||||
if (
|
||||
scoring_func == "sqrtsoftplus"
|
||||
and hash_indices_table is None
|
||||
# dsv4_topk marks padded rows with a -1 sentinel, which unsigned
|
||||
# indices cannot represent; keep the generic path for that pair.
|
||||
and (padding_mask is None or output_indices_dtype != torch.uint32)
|
||||
and can_use_dsv4_topk(
|
||||
gating_output,
|
||||
e_score_correction_bias,
|
||||
@@ -169,7 +165,6 @@ def fused_topk_bias(
|
||||
e_score_correction_bias,
|
||||
output_indices_dtype,
|
||||
routed_scaling_factor,
|
||||
is_padding=padding_mask,
|
||||
input_ids=input_tokens,
|
||||
bias_vl=bias_vl,
|
||||
image_sentinel_lo=image_sentinel_lo,
|
||||
|
||||
Reference in New Issue
Block a user