[Kernels] MoE refactor (#19636)
Signed-off-by: Bill Nell <bnell@redhat.com> Signed-off-by: ElizaWszola <ewszola@redhat.com> Co-authored-by: ElizaWszola <ewszola@redhat.com>
This commit is contained in:
@@ -37,6 +37,7 @@ def _fp8_quantize(
|
||||
A, A_scale = ops.scaled_fp8_quant(
|
||||
A, A_scale, use_per_token_if_dynamic=per_act_token)
|
||||
else:
|
||||
assert not per_act_token
|
||||
assert len(block_shape) == 2
|
||||
_, block_k = block_shape[0], block_shape[1]
|
||||
A, A_scale = per_token_group_quant_fp8(A, block_k)
|
||||
@@ -64,6 +65,7 @@ def _int8_quantize(
|
||||
"int8 quantization only supports block or channel-wise"
|
||||
A, A_scale = per_token_quant_int8(A)
|
||||
else:
|
||||
assert not per_act_token
|
||||
assert len(block_shape) == 2
|
||||
_, block_k = block_shape[0], block_shape[1]
|
||||
A, A_scale = per_token_group_quant_int8(A, block_k)
|
||||
@@ -75,16 +77,15 @@ def _int8_quantize(
|
||||
def moe_kernel_quantize_input(
|
||||
A: torch.Tensor,
|
||||
A_scale: Optional[torch.Tensor],
|
||||
qtype: Optional[torch.dtype],
|
||||
per_channel_quant: bool,
|
||||
quant_dtype: Optional[torch.dtype],
|
||||
per_act_token_quant: bool,
|
||||
block_shape: Optional[list[int]] = None,
|
||||
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
if qtype == torch.float8_e4m3fn:
|
||||
return _fp8_quantize(A, A_scale, per_channel_quant, block_shape)
|
||||
elif qtype == torch.int8:
|
||||
return _int8_quantize(A, A_scale, per_channel_quant, block_shape)
|
||||
if quant_dtype == torch.float8_e4m3fn:
|
||||
return _fp8_quantize(A, A_scale, per_act_token_quant, block_shape)
|
||||
elif quant_dtype == torch.int8:
|
||||
return _int8_quantize(A, A_scale, per_act_token_quant, block_shape)
|
||||
else:
|
||||
assert A_scale is None
|
||||
return A, A_scale
|
||||
|
||||
|
||||
@@ -96,3 +97,17 @@ def _fp8_perm(m: torch.Tensor, idx: torch.Tensor) -> torch.Tensor:
|
||||
return m.view(dtype=torch.uint8)[idx, ...].view(dtype=m.dtype)
|
||||
else:
|
||||
return m[idx, ...]
|
||||
|
||||
|
||||
# TODO(bnell): better name
|
||||
def maybe_fix_scales(scales: Optional[torch.Tensor],
|
||||
num_experts: int) -> Optional[torch.Tensor]:
|
||||
if scales is not None and scales.ndim < 3:
|
||||
if scales.numel() == 1:
|
||||
scales = scales.view(1)
|
||||
scales = torch.repeat_interleave(scales, num_experts,
|
||||
dim=0).view(num_experts, 1, 1)
|
||||
else:
|
||||
scales = scales.view(num_experts, -1, scales.size(-1))
|
||||
|
||||
return scales
|
||||
|
||||
Reference in New Issue
Block a user