[Quantization][Humming] Support MXFP4 weight + block-FP8 activation for MoE (#51332)
Signed-off-by: Elvir Crncevic <elvircrn@gmail.com> Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Signed-off-by: Elvir Crnčević <elvircrn@gmail.com> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
Tyler Michael Smith
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
9c8e90eb26
commit
ca29cd48ba
@@ -611,7 +611,15 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
// SituGLU implementation used in Kimi models.
|
||||
ops.def(
|
||||
"situ_and_mul(Tensor! out, Tensor input, float beta=1.0, float "
|
||||
"linear_beta=-1.0) -> ()");
|
||||
"linear_beta=-1.0, Tensor? valid_rows=None) -> ()");
|
||||
// Fused SituGLU activation + dynamic FP8 quantization for the Humming w2 path
|
||||
// (writes the fp8 down input and its float32 scale). group_size=0 ->
|
||||
// per-token scale [.., 1]; group_size=128 -> k-major block-FP8 scale [..,
|
||||
// d/128].
|
||||
ops.def(
|
||||
"situ_and_mul_quant(Tensor! out, Tensor! scale, Tensor input, "
|
||||
"float beta=1.0, float linear_beta=-1.0, int group_size=0, "
|
||||
"Tensor? valid_rows=None, int topk=1) -> ()");
|
||||
ops.def(
|
||||
"masked_situ_and_mul(Tensor! out, Tensor input, Tensor "
|
||||
"expert_num_tokens, float beta=1.0, float linear_beta=-1.0) -> ()");
|
||||
@@ -830,6 +838,7 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
|
||||
ops.impl("fatrelu_and_mul", TORCH_BOX(&fatrelu_and_mul));
|
||||
ops.impl("swigluoai_and_mul", TORCH_BOX(&swigluoai_and_mul));
|
||||
ops.impl("situ_and_mul", TORCH_BOX(&situ_and_mul));
|
||||
ops.impl("situ_and_mul_quant", TORCH_BOX(&situ_and_mul_quant));
|
||||
ops.impl("masked_situ_and_mul", TORCH_BOX(&masked_situ_and_mul));
|
||||
ops.impl("gelu_new", TORCH_BOX(&gelu_new));
|
||||
ops.impl("gelu_fast", TORCH_BOX(&gelu_fast));
|
||||
|
||||
Reference in New Issue
Block a user