[Perf][MoE] Skip padded work in block-FP8 DeepGEMM experts (#59128)
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
b6d8e8afd9
commit
5ab4e4445b
@@ -158,6 +158,153 @@ def make_block_quant_fp8_weights(
|
|||||||
return w1, w2, w1_s, w2_s
|
return w1, w2, w1_s, w2_s
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("num_tokens", [16, 129, 512])
|
||||||
|
@pytest.mark.parametrize("with_metadata", [False, True])
|
||||||
|
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
|
||||||
|
def test_block_fp8_padding_on_graph_replay(num_tokens, with_metadata, monkeypatch):
|
||||||
|
"""Both FP8 GEMMs consume only live rows as routing changes on replay."""
|
||||||
|
from unittest.mock import Mock
|
||||||
|
|
||||||
|
import vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe as impl
|
||||||
|
from vllm.utils.deep_gemm import DeepGemmQuantScaleFMT
|
||||||
|
|
||||||
|
if not (
|
||||||
|
impl.current_platform.is_device_capability_family(90)
|
||||||
|
or (
|
||||||
|
impl.current_platform.is_device_capability_family(100)
|
||||||
|
and DeepGemmQuantScaleFMT.from_oracle() == DeepGemmQuantScaleFMT.UE8M0
|
||||||
|
)
|
||||||
|
):
|
||||||
|
pytest.skip("Prefix-sum FP8 dispatch requires SM90 or SM10.x")
|
||||||
|
|
||||||
|
torch.manual_seed(0)
|
||||||
|
e, n, k, topk = 8, 1024, 512, 2
|
||||||
|
x = torch.randn(num_tokens, k, device="cuda", dtype=torch.bfloat16) / 10
|
||||||
|
aq, scales = per_token_group_quant_fp8(x, 128, use_ue8m0=True)
|
||||||
|
w1, w2, s1, s2 = make_block_quant_fp8_weights(e, n, k, BLOCK_SIZE)
|
||||||
|
quant = fp8_w8a8_moe_quant_config(w1_scale=s1, w2_scale=s2, block_shape=BLOCK_SIZE)
|
||||||
|
experts = impl.DeepGemmExperts(make_dummy_moe_config(), quant)
|
||||||
|
ids = torch.zeros(num_tokens, topk, device="cuda", dtype=torch.int64)
|
||||||
|
weights = torch.full((num_tokens, topk), 0.5, device="cuda")
|
||||||
|
counts = torch.zeros(e, device="cuda", dtype=torch.int32)
|
||||||
|
metadata = mk.ExpertTokensMetadata(counts, None) if with_metadata else None
|
||||||
|
shapes = experts.workspace_shapes(
|
||||||
|
num_tokens, 2 * n, k, topk, e, e, metadata, MoEActivation.SILU
|
||||||
|
)
|
||||||
|
|
||||||
|
def route(mode):
|
||||||
|
ids[:, 0] = torch.arange(num_tokens, device="cuda") % e
|
||||||
|
ids[:, 1] = (ids[:, 0] + 1) % e
|
||||||
|
if mode == 1:
|
||||||
|
ids.fill_(-1)
|
||||||
|
ids[:1, 0] = e - 1
|
||||||
|
elif mode == 2:
|
||||||
|
ids.fill_(-1)
|
||||||
|
counts.copy_(torch.bincount(ids[ids >= 0], minlength=e).int())
|
||||||
|
|
||||||
|
route(0)
|
||||||
|
variants = []
|
||||||
|
for dense in (True, False):
|
||||||
|
ws1, ws2, output = [
|
||||||
|
torch.empty(shape, device="cuda", dtype=torch.bfloat16) for shape in shapes
|
||||||
|
]
|
||||||
|
|
||||||
|
def invoke(output=output, ws1=ws1, ws2=ws2):
|
||||||
|
experts.apply(
|
||||||
|
output,
|
||||||
|
aq,
|
||||||
|
w1,
|
||||||
|
w2,
|
||||||
|
weights,
|
||||||
|
ids,
|
||||||
|
MoEActivation.SILU,
|
||||||
|
e,
|
||||||
|
None,
|
||||||
|
scales,
|
||||||
|
None,
|
||||||
|
ws1,
|
||||||
|
ws2,
|
||||||
|
metadata,
|
||||||
|
False,
|
||||||
|
)
|
||||||
|
|
||||||
|
with monkeypatch.context() as mp:
|
||||||
|
if dense:
|
||||||
|
platform = Mock(wraps=impl.current_platform)
|
||||||
|
platform.is_device_capability_family.return_value = False
|
||||||
|
mp.setattr(impl, "current_platform", platform)
|
||||||
|
gemm = impl.m_grouped_fp8_gemm_nt_contiguous
|
||||||
|
calls: list[int] = []
|
||||||
|
|
||||||
|
def check_layout(*args, dense=dense, calls=calls, gemm=gemm, **kwargs):
|
||||||
|
assert kwargs.get("use_psum_layout", False) == (not dense)
|
||||||
|
calls.append(args[3].numel())
|
||||||
|
return gemm(*args, **kwargs)
|
||||||
|
|
||||||
|
mp.setattr(impl, "m_grouped_fp8_gemm_nt_contiguous", check_layout)
|
||||||
|
invoke()
|
||||||
|
assert len(calls) == 2
|
||||||
|
if not dense:
|
||||||
|
assert calls == [e, e]
|
||||||
|
graph = torch.cuda.CUDAGraph()
|
||||||
|
with torch.cuda.graph(graph):
|
||||||
|
invoke()
|
||||||
|
variants.append((graph, output, ws1, ws2))
|
||||||
|
|
||||||
|
for mode in (0, 1, 2, 0):
|
||||||
|
route(mode)
|
||||||
|
for graph, _, ws1, ws2 in variants:
|
||||||
|
ws1.fill_(float("nan"))
|
||||||
|
ws2.fill_(float("nan"))
|
||||||
|
graph.replay()
|
||||||
|
torch.accelerator.synchronize()
|
||||||
|
torch.testing.assert_close(variants[0][1], variants[1][1], rtol=0, atol=0)
|
||||||
|
assert torch.isfinite(variants[1][1]).all()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(("n", "k"), [(2048, 512), (512, 1024)])
|
||||||
|
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
|
||||||
|
def test_deepgemm_prefix_sum_leaves_unused_capacity_untouched(n, k):
|
||||||
|
"""Both projection shapes skip unused tiles as counts change on replay."""
|
||||||
|
from vllm.platforms import current_platform
|
||||||
|
from vllm.utils.deep_gemm import (
|
||||||
|
m_grouped_fp8_gemm_nt_contiguous,
|
||||||
|
mk_alignment_scope,
|
||||||
|
)
|
||||||
|
|
||||||
|
if not (
|
||||||
|
current_platform.is_device_capability_family(90)
|
||||||
|
or current_platform.is_device_capability_family(100)
|
||||||
|
):
|
||||||
|
pytest.skip("Requires SM90 or SM10.x")
|
||||||
|
|
||||||
|
capacity, experts, alignment = 512, 4, 128
|
||||||
|
x = torch.randn(capacity, k, device="cuda", dtype=torch.bfloat16) / 10
|
||||||
|
aq, scales = per_token_group_quant_fp8(x, 128, use_ue8m0=True)
|
||||||
|
w, _, ws, _ = make_block_quant_fp8_weights(experts, n // 2, k, BLOCK_SIZE)
|
||||||
|
ends = torch.tensor([0, 1, 128, 129], device="cuda", dtype=torch.int32)
|
||||||
|
output = torch.empty(capacity, n, device="cuda", dtype=torch.bfloat16)
|
||||||
|
|
||||||
|
def invoke():
|
||||||
|
with mk_alignment_scope(alignment):
|
||||||
|
m_grouped_fp8_gemm_nt_contiguous(
|
||||||
|
(aq, scales), (w, ws), output, ends, use_psum_layout=True
|
||||||
|
)
|
||||||
|
|
||||||
|
invoke()
|
||||||
|
graph = torch.cuda.CUDAGraph()
|
||||||
|
with torch.cuda.graph(graph):
|
||||||
|
invoke()
|
||||||
|
for bounds, allocated_end in (([0, 1, 128, 129], 256), ([0] * 4, 0)):
|
||||||
|
ends.copy_(torch.tensor(bounds, device="cuda", dtype=torch.int32))
|
||||||
|
output.fill_(91)
|
||||||
|
graph.replay()
|
||||||
|
torch.accelerator.synchronize()
|
||||||
|
assert torch.all(output[allocated_end:] == 91)
|
||||||
|
if allocated_end:
|
||||||
|
assert torch.all(output[[0, 128]] != 91)
|
||||||
|
|
||||||
|
|
||||||
def run_single_case(m, n, k, topk, num_experts, block_size):
|
def run_single_case(m, n, k, topk, num_experts, block_size):
|
||||||
"""Run one (M,N,K) configuration on a single GPU and assert DeepGEMM ==
|
"""Run one (M,N,K) configuration on a single GPU and assert DeepGEMM ==
|
||||||
Triton baseline within tolerance.
|
Triton baseline within tolerance.
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import torch
|
|||||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||||
_per_token_group_quant_fp8_colmajor,
|
_per_token_group_quant_fp8_colmajor,
|
||||||
silu_mul_per_token_group_quant_fp8_colmajor,
|
silu_mul_per_token_group_quant_fp8_colmajor,
|
||||||
|
silu_mul_quant_fp8_packed_triton,
|
||||||
)
|
)
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
from vllm.triton_utils import triton
|
from vllm.triton_utils import triton
|
||||||
@@ -108,6 +109,94 @@ def test_silu_mul_fp8_quant_deep_gemm(T: int, N: int):
|
|||||||
torch.testing.assert_close(output_scales, ref_output_scales)
|
torch.testing.assert_close(output_scales, ref_output_scales)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"counts", [[0, 1, 127, 128, 129, 0, 257, 13], [128] * 8, [0] * 8]
|
||||||
|
)
|
||||||
|
@pytest.mark.parametrize("group_size", [32, 128])
|
||||||
|
@pytest.mark.parametrize("scale_format", ["packed", "float32", "ceil_ue8m0"])
|
||||||
|
@pytest.mark.parametrize("hidden_size", [256, 2048])
|
||||||
|
@pytest.mark.parametrize("clamp_limit", [None, 10.0])
|
||||||
|
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA graph test")
|
||||||
|
def test_silu_quant_skips_expert_padding_on_graph_replay(
|
||||||
|
counts: list[int],
|
||||||
|
group_size: int,
|
||||||
|
scale_format: str,
|
||||||
|
hidden_size: int,
|
||||||
|
clamp_limit: float | None,
|
||||||
|
):
|
||||||
|
"""Live rows match dense quantization; gaps stay untouched as counts change."""
|
||||||
|
alignment = 128
|
||||||
|
capacity = sum(triton.cdiv(n, alignment) * alignment for n in counts)
|
||||||
|
# Exercise unused capacity as well as the gaps between experts.
|
||||||
|
capacity = max(capacity, alignment)
|
||||||
|
if counts != [128] * 8:
|
||||||
|
capacity += alignment
|
||||||
|
x = torch.randn((capacity, 2 * hidden_size), device=DEVICE, dtype=torch.bfloat16)
|
||||||
|
x *= 16
|
||||||
|
clean_x = x.clone()
|
||||||
|
ends = torch.empty(len(counts), device=DEVICE, dtype=torch.int32)
|
||||||
|
output = torch.empty((capacity, hidden_size), device=DEVICE, dtype=FLOAT8_DTYPE)
|
||||||
|
|
||||||
|
def set_counts(values):
|
||||||
|
positions = []
|
||||||
|
mask = torch.zeros(capacity, device=DEVICE, dtype=torch.bool)
|
||||||
|
start = 0
|
||||||
|
for count in values:
|
||||||
|
mask[start : start + count] = True
|
||||||
|
positions.append(start + count)
|
||||||
|
start += triton.cdiv(count, alignment) * alignment
|
||||||
|
ends.copy_(torch.tensor(positions, device=DEVICE, dtype=torch.int32))
|
||||||
|
return mask
|
||||||
|
|
||||||
|
def quantize(x, output=None, **kwargs):
|
||||||
|
if scale_format == "packed":
|
||||||
|
return silu_mul_quant_fp8_packed_triton(
|
||||||
|
x,
|
||||||
|
group_size=group_size,
|
||||||
|
output_q=output,
|
||||||
|
clamp_limit=clamp_limit,
|
||||||
|
**kwargs,
|
||||||
|
)
|
||||||
|
return silu_mul_per_token_group_quant_fp8_colmajor(
|
||||||
|
x,
|
||||||
|
group_size=group_size,
|
||||||
|
output=output,
|
||||||
|
clamp_limit=clamp_limit,
|
||||||
|
use_ue8m0=scale_format == "ceil_ue8m0",
|
||||||
|
**kwargs,
|
||||||
|
)
|
||||||
|
|
||||||
|
def invoke():
|
||||||
|
return quantize(
|
||||||
|
x,
|
||||||
|
output=output,
|
||||||
|
expert_ends=ends,
|
||||||
|
expert_alignment=alignment,
|
||||||
|
)
|
||||||
|
|
||||||
|
set_counts(counts)
|
||||||
|
invoke()
|
||||||
|
graph = torch.cuda.CUDAGraph()
|
||||||
|
with torch.cuda.graph(graph):
|
||||||
|
actual_q, actual_s = invoke()
|
||||||
|
|
||||||
|
for values in (counts, counts[::-1]):
|
||||||
|
live = set_counts(values)
|
||||||
|
x.copy_(clean_x)
|
||||||
|
x[~live] = float("nan")
|
||||||
|
output.view(torch.uint8).fill_(91)
|
||||||
|
graph.replay()
|
||||||
|
reference_q, reference_s = quantize(x)
|
||||||
|
torch.testing.assert_close(
|
||||||
|
actual_q.view(torch.uint8)[live],
|
||||||
|
reference_q.view(torch.uint8)[live],
|
||||||
|
rtol=0,
|
||||||
|
atol=0,
|
||||||
|
)
|
||||||
|
torch.testing.assert_close(actual_s[live], reference_s[live], rtol=0, atol=0)
|
||||||
|
assert torch.all(actual_q.view(torch.uint8)[~live] == 91)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("T", [128, 256, 512])
|
@pytest.mark.parametrize("T", [128, 256, 512])
|
||||||
@pytest.mark.parametrize("N", [128 * 2, 256 * 2, 768 * 2, 2048 * 2, 7168 * 2])
|
@pytest.mark.parametrize("N", [128 * 2, 256 * 2, 768 * 2, 2048 * 2, 7168 * 2])
|
||||||
@pytest.mark.parametrize("clamp_limit", [7.0, 10.0])
|
@pytest.mark.parametrize("clamp_limit", [7.0, 10.0])
|
||||||
|
|||||||
@@ -119,6 +119,7 @@ def _fwd_kernel_ep_scatter_1(
|
|||||||
BLOCK_E: tl.constexpr,
|
BLOCK_E: tl.constexpr,
|
||||||
BLOCK_EXPERT_NUM: tl.constexpr,
|
BLOCK_EXPERT_NUM: tl.constexpr,
|
||||||
ALIGN_M: tl.constexpr,
|
ALIGN_M: tl.constexpr,
|
||||||
|
USE_PSUM_LAYOUT: tl.constexpr,
|
||||||
):
|
):
|
||||||
cur_expert = tl.program_id(0)
|
cur_expert = tl.program_id(0)
|
||||||
|
|
||||||
@@ -140,20 +141,22 @@ def _fwd_kernel_ep_scatter_1(
|
|||||||
tl.store(expert_start_loc + cur_expert, cur_expert_start)
|
tl.store(expert_start_loc + cur_expert, cur_expert_start)
|
||||||
cur_expert_token_num = tl.load(num_recv_tokens_per_expert + cur_expert)
|
cur_expert_token_num = tl.load(num_recv_tokens_per_expert + cur_expert)
|
||||||
|
|
||||||
m_indices_start_ptr = m_indices + cur_expert_start
|
if USE_PSUM_LAYOUT:
|
||||||
off_expert = tl.arange(0, BLOCK_E)
|
# Logical ends include preceding experts' alignment gaps.
|
||||||
|
tl.store(m_indices + cur_expert, cur_expert_start + cur_expert_token_num)
|
||||||
|
else:
|
||||||
|
m_indices_start_ptr = m_indices + cur_expert_start
|
||||||
|
off_expert = tl.arange(0, BLOCK_E)
|
||||||
|
|
||||||
# any rows in the per-expert aligned region that do not correspond to
|
# Unwritten padding retains the -1 initialization.
|
||||||
# real tokens are left untouched here and should remain initialized to
|
for start_m in tl.range(0, cur_expert_token_num, BLOCK_E):
|
||||||
# -1 so DeepGEMM can skip them
|
offs = start_m + off_expert
|
||||||
for start_m in tl.range(0, cur_expert_token_num, BLOCK_E):
|
mask = offs < cur_expert_token_num
|
||||||
offs = start_m + off_expert
|
tl.store(
|
||||||
mask = offs < cur_expert_token_num
|
m_indices_start_ptr + offs,
|
||||||
tl.store(
|
cur_expert,
|
||||||
m_indices_start_ptr + offs,
|
mask=mask,
|
||||||
cur_expert,
|
)
|
||||||
mask=mask,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@triton.jit
|
@triton.jit
|
||||||
@@ -285,6 +288,7 @@ def ep_scatter(
|
|||||||
align_m: int = 128,
|
align_m: int = 128,
|
||||||
block_size: int = 128,
|
block_size: int = 128,
|
||||||
pack_ue8m0: bool = False,
|
pack_ue8m0: bool = False,
|
||||||
|
use_psum_layout: bool = False,
|
||||||
):
|
):
|
||||||
# BLOCK_E is the m_indices fill-loop tile (masked), independent of align_m.
|
# BLOCK_E is the m_indices fill-loop tile (masked), independent of align_m.
|
||||||
BLOCK_E = 128
|
BLOCK_E = 128
|
||||||
@@ -295,7 +299,10 @@ def ep_scatter(
|
|||||||
# grid = (triton.cdiv(hidden_size, BLOCK_D), num_experts)
|
# grid = (triton.cdiv(hidden_size, BLOCK_D), num_experts)
|
||||||
grid = num_experts
|
grid = num_experts
|
||||||
|
|
||||||
assert m_indices.shape[0] % align_m == 0
|
if use_psum_layout:
|
||||||
|
assert m_indices.shape == (num_experts,)
|
||||||
|
else:
|
||||||
|
assert m_indices.shape[0] % align_m == 0
|
||||||
assert expert_start_loc.shape[0] == num_experts
|
assert expert_start_loc.shape[0] == num_experts
|
||||||
|
|
||||||
# pack_ue8m0: scatter packs 4 UE8M0 bytes per int32; else copies scales as-is.
|
# pack_ue8m0: scatter packs 4 UE8M0 bytes per int32; else copies scales as-is.
|
||||||
@@ -311,6 +318,7 @@ def ep_scatter(
|
|||||||
BLOCK_E=BLOCK_E,
|
BLOCK_E=BLOCK_E,
|
||||||
BLOCK_EXPERT_NUM=triton.next_power_of_2(num_experts),
|
BLOCK_EXPERT_NUM=triton.next_power_of_2(num_experts),
|
||||||
ALIGN_M=align_m,
|
ALIGN_M=align_m,
|
||||||
|
USE_PSUM_LAYOUT=use_psum_layout,
|
||||||
)
|
)
|
||||||
|
|
||||||
grid = min(recv_topk.shape[0], 1024 * 8)
|
grid = min(recv_topk.shape[0], 1024 * 8)
|
||||||
@@ -465,6 +473,7 @@ def deepgemm_moe_permute(
|
|||||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||||
aq_out: torch.Tensor | None = None,
|
aq_out: torch.Tensor | None = None,
|
||||||
block_size: int | None = None,
|
block_size: int | None = None,
|
||||||
|
use_psum_layout: bool = False,
|
||||||
):
|
):
|
||||||
assert aq.ndim == 2
|
assert aq.ndim == 2
|
||||||
assert topk_ids.dtype.is_signed, "The kernel uses -1 to represent invalid topk_ids"
|
assert topk_ids.dtype.is_signed, "The kernel uses -1 to represent invalid topk_ids"
|
||||||
@@ -509,17 +518,15 @@ def deepgemm_moe_permute(
|
|||||||
else:
|
else:
|
||||||
aq_scale_out = torch.zeros((M_sum, sf_k), device=device, dtype=torch.float32)
|
aq_scale_out = torch.zeros((M_sum, sf_k), device=device, dtype=torch.float32)
|
||||||
|
|
||||||
# DeepGEMM uses negative values in m_indices (here expert_ids) to mark
|
if use_psum_layout:
|
||||||
# completely invalid / padded blocks that should be skipped. We always
|
grouped_layout = torch.empty(
|
||||||
# initialize expert_ids to -1 so any row that is not explicitly written
|
(local_num_experts,), device=device, dtype=torch.int32
|
||||||
# by the scatter kernel will be treated as invalid and skipped by
|
)
|
||||||
# DeepGEMM's scheduler.
|
else:
|
||||||
expert_ids = torch.full(
|
# Negative expert IDs mark padding for the contiguous scheduler.
|
||||||
(M_sum,),
|
grouped_layout = torch.full(
|
||||||
fill_value=-1,
|
(M_sum,), fill_value=-1, device=device, dtype=torch.int32
|
||||||
device=device,
|
)
|
||||||
dtype=torch.int32,
|
|
||||||
)
|
|
||||||
inv_perm = torch.empty(topk_ids.shape, device=device, dtype=torch.int32)
|
inv_perm = torch.empty(topk_ids.shape, device=device, dtype=torch.int32)
|
||||||
|
|
||||||
expert_num_tokens = None
|
expert_num_tokens = None
|
||||||
@@ -541,14 +548,15 @@ def deepgemm_moe_permute(
|
|||||||
expert_map=expert_map,
|
expert_map=expert_map,
|
||||||
output_tensor=aq_out,
|
output_tensor=aq_out,
|
||||||
output_tensor_scale=aq_scale_out,
|
output_tensor_scale=aq_scale_out,
|
||||||
m_indices=expert_ids,
|
m_indices=grouped_layout,
|
||||||
output_index=inv_perm,
|
output_index=inv_perm,
|
||||||
align_m=align_used,
|
align_m=align_used,
|
||||||
block_size=block_k,
|
block_size=block_k,
|
||||||
pack_ue8m0=pack_ue8m0,
|
pack_ue8m0=pack_ue8m0,
|
||||||
|
use_psum_layout=use_psum_layout,
|
||||||
)
|
)
|
||||||
|
|
||||||
return aq_out, aq_scale_out, expert_ids, inv_perm, align_used
|
return aq_out, aq_scale_out, grouped_layout, inv_perm, align_used
|
||||||
|
|
||||||
|
|
||||||
def deepgemm_unpermute_and_reduce(
|
def deepgemm_unpermute_and_reduce(
|
||||||
|
|||||||
@@ -244,7 +244,12 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
|||||||
return (workspace1, workspace2, output)
|
return (workspace1, workspace2, output)
|
||||||
|
|
||||||
def _act_mul_quant(
|
def _act_mul_quant(
|
||||||
self, input: torch.Tensor, output: torch.Tensor, activation: MoEActivation
|
self,
|
||||||
|
input: torch.Tensor,
|
||||||
|
output: torch.Tensor,
|
||||||
|
activation: MoEActivation,
|
||||||
|
expert_ends: torch.Tensor | None = None,
|
||||||
|
expert_alignment: int = 0,
|
||||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||||
assert self.block_shape is not None
|
assert self.block_shape is not None
|
||||||
block_k = self.block_shape[1]
|
block_k = self.block_shape[1]
|
||||||
@@ -271,6 +276,8 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
|||||||
clamp_limit=self.gemm1_clamp_limit,
|
clamp_limit=self.gemm1_clamp_limit,
|
||||||
alpha=self.gemm1_alpha,
|
alpha=self.gemm1_alpha,
|
||||||
beta=self.gemm1_beta,
|
beta=self.gemm1_beta,
|
||||||
|
expert_ends=expert_ends,
|
||||||
|
expert_alignment=expert_alignment,
|
||||||
)
|
)
|
||||||
act_out = torch.empty(
|
act_out = torch.empty(
|
||||||
(M_sum, activation_out_dim), dtype=input.dtype, device=input.device
|
(M_sum, activation_out_dim), dtype=input.dtype, device=input.device
|
||||||
@@ -294,6 +301,8 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
|||||||
group_size=block_k,
|
group_size=block_k,
|
||||||
alpha=self.gemm1_alpha,
|
alpha=self.gemm1_alpha,
|
||||||
beta=self.gemm1_beta,
|
beta=self.gemm1_beta,
|
||||||
|
expert_ends=expert_ends,
|
||||||
|
expert_alignment=expert_alignment,
|
||||||
)
|
)
|
||||||
|
|
||||||
# 3. fallback path for non-SiLU activations in non‑UE8M0 cases.
|
# 3. fallback path for non-SiLU activations in non‑UE8M0 cases.
|
||||||
@@ -349,7 +358,20 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
|||||||
a1q_perm = _resize_cache(
|
a1q_perm = _resize_cache(
|
||||||
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, K)
|
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, K)
|
||||||
)
|
)
|
||||||
a1q, a1q_scale, expert_ids, inv_perm, align_used = deepgemm_moe_permute(
|
# Both GEMMs and quantization must agree on the live expert ranges.
|
||||||
|
use_psum_layout = (
|
||||||
|
not self.mxfp8
|
||||||
|
and (
|
||||||
|
current_platform.is_device_capability_family(90)
|
||||||
|
or (
|
||||||
|
current_platform.is_device_capability_family(100)
|
||||||
|
and DeepGemmQuantScaleFMT.from_oracle()
|
||||||
|
== DeepGemmQuantScaleFMT.UE8M0
|
||||||
|
)
|
||||||
|
)
|
||||||
|
and activation in (MoEActivation.SILU, MoEActivation.SWIGLUOAI_UNINTERLEAVE)
|
||||||
|
)
|
||||||
|
a1q, a1q_scale, grouped_layout, inv_perm, align_used = deepgemm_moe_permute(
|
||||||
aq=a1q,
|
aq=a1q,
|
||||||
aq_scale=a1q_scale,
|
aq_scale=a1q_scale,
|
||||||
topk_ids=topk_ids,
|
topk_ids=topk_ids,
|
||||||
@@ -360,17 +382,21 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
|||||||
# MXFP8 uses a 32-element activation-scale group (block_shape[1]);
|
# MXFP8 uses a 32-element activation-scale group (block_shape[1]);
|
||||||
# FP8-block keeps the default (128) alignment.
|
# FP8-block keeps the default (128) alignment.
|
||||||
block_size=self.block_shape[1] if self.mxfp8 else None,
|
block_size=self.block_shape[1] if self.mxfp8 else None,
|
||||||
|
use_psum_layout=use_psum_layout,
|
||||||
)
|
)
|
||||||
assert a1q.size(0) == M_sum
|
assert a1q.size(0) == M_sum
|
||||||
|
|
||||||
# MXFP8 (1x32) drives the fp8_fp4-aliased grouped GEMM with recipe
|
# MXFP8 (1x32) drives the fp8_fp4-aliased grouped GEMM with recipe
|
||||||
# (1, 32); the FP8 block path keeps the default (128) recipe.
|
# (1, 32); the FP8 block path keeps the default (128) recipe.
|
||||||
gemm_kwargs = (
|
gemm_kwargs: dict = (
|
||||||
{"recipe_a": (1, self.block_shape[1]), "recipe_b": (1, self.block_shape[1])}
|
{"recipe_a": (1, self.block_shape[1]), "recipe_b": (1, self.block_shape[1])}
|
||||||
if self.mxfp8
|
if self.mxfp8
|
||||||
else {}
|
else {}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if use_psum_layout:
|
||||||
|
gemm_kwargs["use_psum_layout"] = True
|
||||||
|
|
||||||
# Cap DG's BLOCK_M heuristic at the workspace's per-expert alignment;
|
# Cap DG's BLOCK_M heuristic at the workspace's per-expert alignment;
|
||||||
# otherwise the scheduler can pick the wrong expert id from m_indices
|
# otherwise the scheduler can pick the wrong expert id from m_indices
|
||||||
# under cudagraph replay.
|
# under cudagraph replay.
|
||||||
@@ -380,7 +406,7 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
|||||||
(a1q, a1q_scale),
|
(a1q, a1q_scale),
|
||||||
(w1, self.w1_scale),
|
(w1, self.w1_scale),
|
||||||
mm1_out,
|
mm1_out,
|
||||||
expert_ids,
|
grouped_layout,
|
||||||
**gemm_kwargs,
|
**gemm_kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -389,7 +415,11 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
|||||||
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, activation_out_dim)
|
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, activation_out_dim)
|
||||||
)
|
)
|
||||||
a2q, a2q_scale = self._act_mul_quant(
|
a2q, a2q_scale = self._act_mul_quant(
|
||||||
input=mm1_out.view(-1, N), output=quant_out, activation=activation
|
input=mm1_out.view(-1, N),
|
||||||
|
output=quant_out,
|
||||||
|
activation=activation,
|
||||||
|
expert_ends=grouped_layout if use_psum_layout else None,
|
||||||
|
expert_alignment=align_used if use_psum_layout else 0,
|
||||||
)
|
)
|
||||||
|
|
||||||
mm2_out = _resize_cache(workspace2, (M_sum, K))
|
mm2_out = _resize_cache(workspace2, (M_sum, K))
|
||||||
@@ -397,7 +427,7 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
|||||||
(a2q, a2q_scale),
|
(a2q, a2q_scale),
|
||||||
(w2, self.w2_scale),
|
(w2, self.w2_scale),
|
||||||
mm2_out,
|
mm2_out,
|
||||||
expert_ids,
|
grouped_layout,
|
||||||
**gemm_kwargs,
|
**gemm_kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -142,6 +142,7 @@ def _silu_mul_quant_fp8_packed_kernel(
|
|||||||
input_ptr,
|
input_ptr,
|
||||||
output_q_ptr,
|
output_q_ptr,
|
||||||
output_scale_ptr,
|
output_scale_ptr,
|
||||||
|
expert_ends_ptr,
|
||||||
M,
|
M,
|
||||||
input_stride_m,
|
input_stride_m,
|
||||||
output_q_stride_m,
|
output_q_stride_m,
|
||||||
@@ -158,6 +159,7 @@ def _silu_mul_quant_fp8_packed_kernel(
|
|||||||
PACKS_PER_CTA: tl.constexpr,
|
PACKS_PER_CTA: tl.constexpr,
|
||||||
BLOCK_M: tl.constexpr,
|
BLOCK_M: tl.constexpr,
|
||||||
HAS_CLAMP: tl.constexpr,
|
HAS_CLAMP: tl.constexpr,
|
||||||
|
EXPERT_ALIGNMENT: tl.constexpr,
|
||||||
):
|
):
|
||||||
GROUPS_PER_PACK: tl.constexpr = 4
|
GROUPS_PER_PACK: tl.constexpr = 4
|
||||||
hidden_size: tl.constexpr = N // 2
|
hidden_size: tl.constexpr = N // 2
|
||||||
@@ -166,6 +168,13 @@ def _silu_mul_quant_fp8_packed_kernel(
|
|||||||
row_start = tl.program_id(1).to(tl.int64) * BLOCK_M
|
row_start = tl.program_id(1).to(tl.int64) * BLOCK_M
|
||||||
row_step = tl.num_programs(1).to(tl.int64) * BLOCK_M
|
row_step = tl.num_programs(1).to(tl.int64) * BLOCK_M
|
||||||
|
|
||||||
|
if EXPERT_ALIGNMENT:
|
||||||
|
expert = tl.program_id(2)
|
||||||
|
previous_end = tl.load(expert_ends_ptr + expert - 1, expert > 0, other=0)
|
||||||
|
expert_start = tl.cdiv(previous_end, EXPERT_ALIGNMENT) * EXPERT_ALIGNMENT
|
||||||
|
row_start += expert_start.to(tl.int64)
|
||||||
|
M = tl.load(expert_ends_ptr + expert)
|
||||||
|
|
||||||
groups_per_cta: tl.constexpr = PACKS_PER_CTA * GROUPS_PER_PACK
|
groups_per_cta: tl.constexpr = PACKS_PER_CTA * GROUPS_PER_PACK
|
||||||
elems_per_cta: tl.constexpr = groups_per_cta * GROUP_SIZE
|
elems_per_cta: tl.constexpr = groups_per_cta * GROUP_SIZE
|
||||||
col_start = pack_tile * elems_per_cta
|
col_start = pack_tile * elems_per_cta
|
||||||
@@ -254,7 +263,19 @@ def silu_mul_quant_fp8_packed_triton(
|
|||||||
clamp_limit: float | None = None,
|
clamp_limit: float | None = None,
|
||||||
alpha: float = 1.0,
|
alpha: float = 1.0,
|
||||||
beta: float = 0.0,
|
beta: float = 0.0,
|
||||||
|
*,
|
||||||
|
expert_ends: torch.Tensor | None = None,
|
||||||
|
expert_alignment: int = 0,
|
||||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||||
|
"""Fuse activation and FP8 quantization, optionally skipping expert padding.
|
||||||
|
|
||||||
|
``expert_ends`` uses DeepGEMM's prefix-sum layout: each entry is the
|
||||||
|
exclusive end of an expert's live rows, including earlier alignment gaps.
|
||||||
|
An expert starts at the preceding end rounded up to ``expert_alignment``.
|
||||||
|
Padding outputs are left unwritten and must not be consumed as live rows.
|
||||||
|
Without ``expert_ends``, the original dense launch configuration is used
|
||||||
|
and the metadata loads are compiled out.
|
||||||
|
"""
|
||||||
assert input.dim() == 2
|
assert input.dim() == 2
|
||||||
assert input.is_contiguous()
|
assert input.is_contiguous()
|
||||||
|
|
||||||
@@ -293,13 +314,29 @@ def silu_mul_quant_fp8_packed_triton(
|
|||||||
|
|
||||||
grid_n = triton.cdiv(packs_per_row, packs_per_cta)
|
grid_n = triton.cdiv(packs_per_row, packs_per_cta)
|
||||||
grid_m = min(triton.cdiv(M, BM), 4096)
|
grid_m = min(triton.cdiv(M, BM), 4096)
|
||||||
grid = (grid_n, grid_m)
|
grid: tuple[int, ...] = (grid_n, grid_m)
|
||||||
|
if expert_ends is not None:
|
||||||
|
assert expert_ends.ndim == 1 and expert_ends.numel() > 0
|
||||||
|
assert expert_ends.dtype == torch.int32
|
||||||
|
assert expert_ends.device == input.device and expert_ends.is_contiguous()
|
||||||
|
assert expert_alignment > 0
|
||||||
|
num_experts = expert_ends.numel()
|
||||||
|
# Bound empty CTAs while retaining enough parallelism for full experts.
|
||||||
|
max_ctas = 4096 if group_size < 128 else 8192
|
||||||
|
grid_m = min(
|
||||||
|
triton.cdiv(M, BM * num_experts),
|
||||||
|
max(1, triton.cdiv(max_ctas, grid_n * num_experts)),
|
||||||
|
)
|
||||||
|
grid = (grid_n, grid_m, num_experts)
|
||||||
|
else:
|
||||||
|
assert expert_alignment == 0
|
||||||
|
|
||||||
has_clamp = clamp_limit is not None
|
has_clamp = clamp_limit is not None
|
||||||
_silu_mul_quant_fp8_packed_kernel[grid](
|
_silu_mul_quant_fp8_packed_kernel[grid](
|
||||||
input,
|
input,
|
||||||
output_q,
|
output_q,
|
||||||
output_scale_packed,
|
output_scale_packed,
|
||||||
|
expert_ends,
|
||||||
M,
|
M,
|
||||||
input.stride(0),
|
input.stride(0),
|
||||||
output_q.stride(0),
|
output_q.stride(0),
|
||||||
@@ -316,6 +353,7 @@ def silu_mul_quant_fp8_packed_triton(
|
|||||||
PACKS_PER_CTA=packs_per_cta,
|
PACKS_PER_CTA=packs_per_cta,
|
||||||
BLOCK_M=BM,
|
BLOCK_M=BM,
|
||||||
HAS_CLAMP=has_clamp,
|
HAS_CLAMP=has_clamp,
|
||||||
|
EXPERT_ALIGNMENT=expert_alignment,
|
||||||
num_warps=num_warps,
|
num_warps=num_warps,
|
||||||
num_stages=num_stages,
|
num_stages=num_stages,
|
||||||
)
|
)
|
||||||
@@ -328,6 +366,7 @@ def _silu_mul_per_token_group_quant_fp8_colmajor(
|
|||||||
y_ptr, # [M, N]
|
y_ptr, # [M, N]
|
||||||
y_q_ptr, # [M, N // 2]
|
y_q_ptr, # [M, N // 2]
|
||||||
y_s_ptr, # [M, (N // 2) // GROUP_SIZE]
|
y_s_ptr, # [M, (N // 2) // GROUP_SIZE]
|
||||||
|
expert_ends_ptr,
|
||||||
M, # num tokens
|
M, # num tokens
|
||||||
N, # intermediate size
|
N, # intermediate size
|
||||||
# Stride
|
# Stride
|
||||||
@@ -345,8 +384,8 @@ def _silu_mul_per_token_group_quant_fp8_colmajor(
|
|||||||
GROUP_SIZE: tl.constexpr,
|
GROUP_SIZE: tl.constexpr,
|
||||||
BLOCK_M: tl.constexpr,
|
BLOCK_M: tl.constexpr,
|
||||||
BLOCK_N: tl.constexpr,
|
BLOCK_N: tl.constexpr,
|
||||||
|
EXPERT_ALIGNMENT: tl.constexpr,
|
||||||
):
|
):
|
||||||
# TODO(varun) : Add expert_ids so we may early-exit no-op thread blocks.
|
|
||||||
"""Each thread block (BLOCK_N) computes [BLOCK_M, GROUP_SIZE] act-mul outputs. Then
|
"""Each thread block (BLOCK_N) computes [BLOCK_M, GROUP_SIZE] act-mul outputs. Then
|
||||||
the thread block quantizes the [BLOCK_M, GROUP_SIZE] block of values and fills
|
the thread block quantizes the [BLOCK_M, GROUP_SIZE] block of values and fills
|
||||||
the outputs tensors at the right positions.
|
the outputs tensors at the right positions.
|
||||||
@@ -357,56 +396,77 @@ def _silu_mul_per_token_group_quant_fp8_colmajor(
|
|||||||
|
|
||||||
m_offset = pid_m.to(tl.int64) * BLOCK_M
|
m_offset = pid_m.to(tl.int64) * BLOCK_M
|
||||||
n_offset = pid_n.to(tl.int64) * BLOCK_N
|
n_offset = pid_n.to(tl.int64) * BLOCK_N
|
||||||
if m_offset >= M:
|
row_step = tl.num_programs(0).to(tl.int64) * BLOCK_M
|
||||||
return
|
num_iterations = 1
|
||||||
|
if EXPERT_ALIGNMENT:
|
||||||
|
expert = tl.program_id(2)
|
||||||
|
previous_end = tl.load(expert_ends_ptr + expert - 1, expert > 0, other=0)
|
||||||
|
expert_start = tl.cdiv(previous_end, EXPERT_ALIGNMENT) * EXPERT_ALIGNMENT
|
||||||
|
m_offset += expert_start.to(tl.int64)
|
||||||
|
M = tl.load(expert_ends_ptr + expert)
|
||||||
|
num_iterations = tl.cdiv(tl.maximum(M - m_offset, 0), row_step)
|
||||||
|
|
||||||
offs_n = tl.arange(0, BLOCK_N).to(tl.int64)
|
offs_n = tl.arange(0, BLOCK_N).to(tl.int64)
|
||||||
offs_m = tl.arange(0, BLOCK_M).to(tl.int64)
|
offs_m = tl.arange(0, BLOCK_M).to(tl.int64)
|
||||||
|
|
||||||
base_y_ptr = y_ptr + m_offset * N + n_offset
|
for _ in range(num_iterations):
|
||||||
|
base_y_ptr = y_ptr + m_offset * N + n_offset
|
||||||
|
|
||||||
act_in_ptrs = base_y_ptr + offs_m[:, None] * N + offs_n[None, :]
|
act_in_ptrs = base_y_ptr + offs_m[:, None] * N + offs_n[None, :]
|
||||||
|
|
||||||
act_in = tl.load(act_in_ptrs)
|
if EXPERT_ALIGNMENT:
|
||||||
mul_in = tl.load(act_in_ptrs + N_2)
|
live = m_offset + offs_m < M
|
||||||
|
act_in = tl.load(act_in_ptrs, live[:, None], other=0)
|
||||||
|
mul_in = tl.load(act_in_ptrs + N_2, live[:, None], other=0)
|
||||||
|
else:
|
||||||
|
act_in = tl.load(act_in_ptrs)
|
||||||
|
mul_in = tl.load(act_in_ptrs + N_2)
|
||||||
|
|
||||||
# silu & mul — match C++ silu_and_mul: clamp in fp32 then store back to the
|
# silu & mul — match C++ silu_and_mul: clamp in fp32 then store back to the
|
||||||
# input dtype, run silu in fp32 then narrow, and do the mul at input
|
# input dtype, run silu in fp32 then narrow, and do the mul at input
|
||||||
# precision so HAS_CLAMP True/False share the same multiplication path.
|
# precision so HAS_CLAMP True/False share the same multiplication path.
|
||||||
if HAS_CLAMP:
|
if HAS_CLAMP:
|
||||||
act_in = tl.minimum(act_in.to(tl.float32), clamp_limit).to(
|
act_in = tl.minimum(act_in.to(tl.float32), clamp_limit).to(
|
||||||
y_ptr.dtype.element_ty
|
y_ptr.dtype.element_ty
|
||||||
)
|
)
|
||||||
mul_in = tl.clamp(mul_in.to(tl.float32), -clamp_limit, clamp_limit).to(
|
mul_in = tl.clamp(mul_in.to(tl.float32), -clamp_limit, clamp_limit).to(
|
||||||
y_ptr.dtype.element_ty
|
y_ptr.dtype.element_ty
|
||||||
)
|
)
|
||||||
# Unified gated activation: silu == swigluoai with alpha=1, beta=0.
|
# Unified gated activation: silu == swigluoai with alpha=1, beta=0.
|
||||||
# glu = gate * sigmoid(alpha * gate); y = (up + beta) * glu
|
# glu = gate * sigmoid(alpha * gate); y = (up + beta) * glu
|
||||||
# Keep glu/up at input precision (narrow before the mul) so the alpha=1,
|
# Keep glu/up at input precision (narrow before the mul) so the alpha=1,
|
||||||
# beta=0 defaults match the C++ silu_and_mul path bit-for-bit.
|
# beta=0 defaults match the C++ silu_and_mul path bit-for-bit.
|
||||||
act_in = act_in.to(tl.float32)
|
act_in = act_in.to(tl.float32)
|
||||||
glu = (act_in / (1.0 + tl.exp(-act_in * alpha))).to(y_ptr.dtype.element_ty)
|
glu = (act_in / (1.0 + tl.exp(-act_in * alpha))).to(y_ptr.dtype.element_ty)
|
||||||
up = (mul_in.to(tl.float32) + beta).to(y_ptr.dtype.element_ty)
|
up = (mul_in.to(tl.float32) + beta).to(y_ptr.dtype.element_ty)
|
||||||
y = (glu * up).to(tl.float32)
|
y = (glu * up).to(tl.float32)
|
||||||
|
|
||||||
# quant
|
# quant
|
||||||
_absmax = tl.maximum(tl.max(tl.abs(y), axis=1), eps)
|
_absmax = tl.maximum(tl.max(tl.abs(y), axis=1), eps)
|
||||||
scale_raw = _absmax * (1.0 / fp8_max)
|
scale_raw = _absmax * (1.0 / fp8_max)
|
||||||
y_s = tl.math.exp2(tl.ceil(tl.log2(scale_raw))) if use_ue8m0 else scale_raw
|
y_s = tl.math.exp2(tl.ceil(tl.log2(scale_raw))) if use_ue8m0 else scale_raw
|
||||||
y_s = tl.reshape(y_s, (BLOCK_M, 1))
|
y_s = tl.reshape(y_s, (BLOCK_M, 1))
|
||||||
y_q = tl.clamp(y / y_s, fp8_min, fp8_max).to(y_q_ptr.dtype.element_ty)
|
y_q = tl.clamp(y / y_s, fp8_min, fp8_max).to(y_q_ptr.dtype.element_ty)
|
||||||
|
|
||||||
# store y_q
|
# store y_q
|
||||||
base_y_q_ptr = y_q_ptr + m_offset * N_2 + n_offset
|
base_y_q_ptr = y_q_ptr + m_offset * N_2 + n_offset
|
||||||
y_q_ptrs = base_y_q_ptr + offs_m[:, None] * N_2 + offs_n[None, :]
|
y_q_ptrs = base_y_q_ptr + offs_m[:, None] * N_2 + offs_n[None, :]
|
||||||
tl.store(y_q_ptrs, y_q)
|
if EXPERT_ALIGNMENT:
|
||||||
|
tl.store(y_q_ptrs, y_q, live[:, None])
|
||||||
|
else:
|
||||||
|
tl.store(y_q_ptrs, y_q)
|
||||||
|
|
||||||
# store y_s
|
# store y_s
|
||||||
group_id = n_offset // GROUP_SIZE
|
group_id = n_offset // GROUP_SIZE
|
||||||
base_y_s_ptr = y_s_ptr + group_id * y_s_col_stride + m_offset
|
base_y_s_ptr = y_s_ptr + group_id * y_s_col_stride + m_offset
|
||||||
y_s_ptrs = base_y_s_ptr + offs_m
|
y_s_ptrs = base_y_s_ptr + offs_m
|
||||||
y_s = tl.reshape(y_s, (BLOCK_M,))
|
y_s = tl.reshape(y_s, (BLOCK_M,))
|
||||||
tl.store(y_s_ptrs, y_s)
|
if EXPERT_ALIGNMENT:
|
||||||
|
tl.store(y_s_ptrs, y_s, live)
|
||||||
|
else:
|
||||||
|
tl.store(y_s_ptrs, y_s)
|
||||||
|
|
||||||
|
m_offset += row_step
|
||||||
|
|
||||||
|
|
||||||
def silu_mul_per_token_group_quant_fp8_colmajor(
|
def silu_mul_per_token_group_quant_fp8_colmajor(
|
||||||
@@ -418,9 +478,15 @@ def silu_mul_per_token_group_quant_fp8_colmajor(
|
|||||||
group_size: int = 128,
|
group_size: int = 128,
|
||||||
alpha: float = 1.0,
|
alpha: float = 1.0,
|
||||||
beta: float = 0.0,
|
beta: float = 0.0,
|
||||||
|
*,
|
||||||
|
expert_ends: torch.Tensor | None = None,
|
||||||
|
expert_alignment: int = 0,
|
||||||
):
|
):
|
||||||
"""Gated activation + block-fp8 quant. ``alpha``/``beta`` select the gate
|
"""Gated activation + block-fp8 quant. ``alpha``/``beta`` select the gate
|
||||||
(silu: alpha=1, beta=0; swigluoai: alpha, beta from config).
|
(silu: alpha=1, beta=0; swigluoai: alpha, beta from config).
|
||||||
|
|
||||||
|
Optional expert endpoints follow DeepGEMM's prefix-sum layout. Padding
|
||||||
|
values and scales are left unwritten and must not be consumed as live rows.
|
||||||
"""
|
"""
|
||||||
GROUP_SIZE = group_size
|
GROUP_SIZE = group_size
|
||||||
assert input.ndim == 2
|
assert input.ndim == 2
|
||||||
@@ -458,13 +524,27 @@ def silu_mul_per_token_group_quant_fp8_colmajor(
|
|||||||
# Force even division so we can avoid edgecases within the kernel.
|
# Force even division so we can avoid edgecases within the kernel.
|
||||||
assert M % BLOCK_M == 0
|
assert M % BLOCK_M == 0
|
||||||
assert N_2 % BLOCK_N == 0
|
assert N_2 % BLOCK_N == 0
|
||||||
grid = (M // BLOCK_M, N_2 // BLOCK_N)
|
grid: tuple[int, ...] = (M // BLOCK_M, N_2 // BLOCK_N)
|
||||||
|
if expert_ends is not None:
|
||||||
|
assert expert_ends.ndim == 1 and expert_ends.numel() > 0
|
||||||
|
assert expert_ends.dtype == torch.int32
|
||||||
|
assert expert_ends.device == input.device and expert_ends.is_contiguous()
|
||||||
|
assert expert_alignment > 0 and expert_alignment % BLOCK_M == 0
|
||||||
|
num_experts = expert_ends.numel()
|
||||||
|
grid_m = min(
|
||||||
|
triton.cdiv(M, BLOCK_M * num_experts),
|
||||||
|
max(1, triton.cdiv(8192, grid[1] * num_experts)),
|
||||||
|
)
|
||||||
|
grid = (grid_m, grid[1], num_experts)
|
||||||
|
else:
|
||||||
|
assert expert_alignment == 0
|
||||||
|
|
||||||
has_clamp = clamp_limit is not None
|
has_clamp = clamp_limit is not None
|
||||||
_silu_mul_per_token_group_quant_fp8_colmajor[grid](
|
_silu_mul_per_token_group_quant_fp8_colmajor[grid](
|
||||||
input,
|
input,
|
||||||
output,
|
output,
|
||||||
output_scales,
|
output_scales,
|
||||||
|
expert_ends,
|
||||||
M,
|
M,
|
||||||
N,
|
N,
|
||||||
output_scales.stride(-1),
|
output_scales.stride(-1),
|
||||||
@@ -479,6 +559,7 @@ def silu_mul_per_token_group_quant_fp8_colmajor(
|
|||||||
GROUP_SIZE,
|
GROUP_SIZE,
|
||||||
BLOCK_M,
|
BLOCK_M,
|
||||||
BLOCK_N,
|
BLOCK_N,
|
||||||
|
expert_alignment,
|
||||||
)
|
)
|
||||||
|
|
||||||
return output, output_scales
|
return output, output_scales
|
||||||
|
|||||||
Reference in New Issue
Block a user