[Quantization][Humming] Support MXFP4 weight + block-FP8 activation for MoE (#51332)
Signed-off-by: Elvir Crncevic <elvircrn@gmail.com> Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Signed-off-by: Elvir Crnčević <elvircrn@gmail.com> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
Tyler Michael Smith
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
9c8e90eb26
commit
ca29cd48ba
@@ -0,0 +1,11 @@
|
||||
model_name: "nm-testing/Qwen3-30B-A3B-MXFP4A16"
|
||||
accuracy_threshold: 0.86
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--quantization humming
|
||||
--linear-backend humming
|
||||
env:
|
||||
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3","group_size":128}'
|
||||
@@ -1,5 +1,6 @@
|
||||
gpt-oss-20b-humming-act-fp8.yaml
|
||||
Qwen3-30B-A3B-MXFP4A16-humming-act-fp8.yaml
|
||||
Qwen3-30B-A3B-MXFP4A16-humming-act-fp8-block.yaml
|
||||
Qwen2-1.5B-Instruct-FP8W8-humming-act-fp8.yaml
|
||||
Qwen3-0.6B-MXFP8-humming-act-fp8.yaml
|
||||
Qwen3-30B-A3B-Fp8-v1-humming-act-fp8.yaml
|
||||
|
||||
@@ -1408,6 +1408,7 @@ def test_humming_delegates_to_instance_activation():
|
||||
activation=MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
input=input,
|
||||
output=output,
|
||||
valid_rows=None,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,406 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the fused MoE topk weight-and-reduce kernel.
|
||||
|
||||
Run `pytest tests/kernels/moe/test_moe_fused_mul_sum.py`.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.moe_fused_mul_sum import (
|
||||
moe_fused_mul_sum,
|
||||
)
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
NUM_TOKENS = [1, 17, 256]
|
||||
TOP_KS = [2, 8]
|
||||
HIDDEN_SIZE = 512
|
||||
# One BLOCK_K tile, a partial trailing tile, and Kimi-K3's real dims
|
||||
# (moe_intermediate_size 3072, hidden_size 7168) -- exercises the per-tile loop
|
||||
# and its tail mask (BLOCK_K caps at 512 for <=2-byte dtypes).
|
||||
HIDDEN_SIZES = [512, 1000, 3072, 7168]
|
||||
DTYPES = [torch.bfloat16, torch.float32]
|
||||
NUM_EXPERTS = 16
|
||||
|
||||
|
||||
def _reference(
|
||||
inputs: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
expert_map: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
valid = topk_ids >= 0
|
||||
if expert_map is not None:
|
||||
valid &= expert_map[topk_ids.clamp(min=0)] >= 0
|
||||
weights = torch.where(valid, topk_weights, torch.zeros_like(topk_weights))
|
||||
return (inputs.float() * weights.float().unsqueeze(-1)).sum(dim=1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("top_k", TOP_KS)
|
||||
@pytest.mark.parametrize("use_expert_map", [True, False])
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_invalid_slots_excluded(num_tokens: int, top_k: int, use_expert_map: bool):
|
||||
"""Slots marked -1 must not contribute, and must not be dereferenced.
|
||||
|
||||
The expert GEMM never writes rows for topk slots whose expert id is -1
|
||||
(a non-local expert under EP, or an all2all padding row), so those rows
|
||||
hold stale workspace data. Filling them with NaN here means any failure to
|
||||
mask them shows up as NaN in the output rather than a small numeric drift.
|
||||
Indexing `expert_map` with -1 is also an out-of-bounds read.
|
||||
"""
|
||||
set_random_seed(0)
|
||||
device = "cuda"
|
||||
|
||||
inputs = torch.randn(
|
||||
num_tokens, top_k, HIDDEN_SIZE, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
topk_weights = torch.rand(num_tokens, top_k, dtype=torch.bfloat16, device=device)
|
||||
topk_ids = torch.randint(
|
||||
0, NUM_EXPERTS, (num_tokens, top_k), dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
# Mark roughly half the slots invalid and poison the rows behind them.
|
||||
invalid = torch.rand(num_tokens, top_k, device=device) < 0.5
|
||||
topk_ids[invalid] = -1
|
||||
inputs[invalid] = float("nan")
|
||||
|
||||
expert_map = None
|
||||
if use_expert_map:
|
||||
# Second half of the experts is non-local, so valid ids can still map
|
||||
# to -1 -- exercising the second masking condition.
|
||||
expert_map = torch.full((NUM_EXPERTS,), -1, dtype=torch.int32, device=device)
|
||||
local = NUM_EXPERTS // 2
|
||||
expert_map[:local] = torch.arange(local, dtype=torch.int32, device=device)
|
||||
inputs[(topk_ids >= 0) & (expert_map[topk_ids.clamp(min=0)] < 0)] = float("nan")
|
||||
|
||||
# The _reference implementation zeros out outputs. We need to match that here,
|
||||
# otherwise the test fails.
|
||||
outputs = torch.zeros(num_tokens, HIDDEN_SIZE, dtype=torch.bfloat16, device=device)
|
||||
out = moe_fused_mul_sum(
|
||||
inputs=inputs,
|
||||
topk_weights=topk_weights,
|
||||
outputs=outputs,
|
||||
topk_ids=topk_ids,
|
||||
expert_map=expert_map,
|
||||
)
|
||||
|
||||
assert not out.isnan().any(), "invalid slots leaked into the reduction"
|
||||
ref = _reference(inputs.nan_to_num(), topk_weights, topk_ids, expert_map)
|
||||
torch.testing.assert_close(out.float(), ref, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
def _local_expert_map(device: str) -> torch.Tensor:
|
||||
"""First half of the experts is local, second half maps to -1 (non-local)."""
|
||||
expert_map = torch.full((NUM_EXPERTS,), -1, dtype=torch.int32, device=device)
|
||||
local = NUM_EXPERTS // 2
|
||||
expert_map[:local] = torch.arange(local, dtype=torch.int32, device=device)
|
||||
return expert_map
|
||||
|
||||
|
||||
SENTINEL = 42.0 # exact in bf16; marks output the kernel must leave untouched
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_nonlocal_row_is_zeroed():
|
||||
"""A real row whose positive ids all map to non-local experts is written 0.
|
||||
|
||||
Such a row is not padding (its ids are >= 0, so row_valid > 0 and the block
|
||||
is not early-returned). Every slot is then masked out by expert_map, so the
|
||||
accumulator stays 0 and 0 is stored -- the row must not be left holding stale
|
||||
buffer data, because downstream reduction treats it as a real token.
|
||||
"""
|
||||
set_random_seed(0)
|
||||
device = "cuda"
|
||||
num_tokens, top_k = 17, 8
|
||||
local = NUM_EXPERTS // 2
|
||||
expert_map = _local_expert_map(device)
|
||||
|
||||
# Poison inputs so any failure to mask the non-local slots surfaces as NaN.
|
||||
inputs = torch.full(
|
||||
(num_tokens, top_k, HIDDEN_SIZE),
|
||||
float("nan"),
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
topk_weights = torch.rand(num_tokens, top_k, dtype=torch.bfloat16, device=device)
|
||||
# Every id lands in the non-local half [local, NUM_EXPERTS).
|
||||
topk_ids = torch.randint(
|
||||
local, NUM_EXPERTS, (num_tokens, top_k), dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
outputs = torch.full(
|
||||
(num_tokens, HIDDEN_SIZE), SENTINEL, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
out = moe_fused_mul_sum(
|
||||
inputs=inputs,
|
||||
topk_weights=topk_weights,
|
||||
outputs=outputs,
|
||||
topk_ids=topk_ids,
|
||||
expert_map=expert_map,
|
||||
)
|
||||
|
||||
assert not out.isnan().any(), "non-local row leaked NaN into the output"
|
||||
torch.testing.assert_close(out.float(), torch.zeros_like(out.float()))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_expert_map", [True, False])
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_padding_row_left_untouched(use_expert_map: bool):
|
||||
"""An all -1 padding row is skipped: its output row is left untouched.
|
||||
|
||||
This is the CUDA-graph decode contract -- padding rows past num_recv are
|
||||
never read downstream, so the kernel elides them (whole-padding blocks
|
||||
early-return) rather than zeroing. The skip is gated on topk_ids alone, so
|
||||
it must hold whether or not expert_map is passed (the humming path passes
|
||||
None); we pin both.
|
||||
"""
|
||||
set_random_seed(0)
|
||||
device = "cuda"
|
||||
num_tokens, top_k = 17, 8
|
||||
expert_map = _local_expert_map(device) if use_expert_map else None
|
||||
|
||||
inputs = torch.full(
|
||||
(num_tokens, top_k, HIDDEN_SIZE),
|
||||
float("nan"),
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
topk_weights = torch.rand(num_tokens, top_k, dtype=torch.bfloat16, device=device)
|
||||
topk_ids = torch.full((num_tokens, top_k), -1, dtype=torch.int32, device=device)
|
||||
|
||||
outputs = torch.full(
|
||||
(num_tokens, HIDDEN_SIZE), SENTINEL, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
out = moe_fused_mul_sum(
|
||||
inputs=inputs,
|
||||
topk_weights=topk_weights,
|
||||
outputs=outputs,
|
||||
topk_ids=topk_ids,
|
||||
expert_map=expert_map,
|
||||
)
|
||||
|
||||
assert torch.all(out == SENTINEL), "padding row output was modified"
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_nonlocal_and_padding_adjacent():
|
||||
"""A non-local real row next to an all -1 padding row.
|
||||
|
||||
Each CTA owns one token row: the non-local row must be zeroed (its slots are
|
||||
all masked by expert_map) while the padding row beside it is left untouched.
|
||||
Guards against a change that lets one row's fate leak into its neighbour.
|
||||
"""
|
||||
set_random_seed(0)
|
||||
device = "cuda"
|
||||
num_tokens, top_k = 17, 8
|
||||
local = NUM_EXPERTS // 2
|
||||
expert_map = _local_expert_map(device)
|
||||
|
||||
inputs = torch.full(
|
||||
(num_tokens, top_k, HIDDEN_SIZE),
|
||||
float("nan"),
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
topk_weights = torch.rand(num_tokens, top_k, dtype=torch.bfloat16, device=device)
|
||||
# All rows are non-local real rows except row 1, an all -1 padding row.
|
||||
topk_ids = torch.randint(
|
||||
local, NUM_EXPERTS, (num_tokens, top_k), dtype=torch.int32, device=device
|
||||
)
|
||||
topk_ids[1] = -1
|
||||
|
||||
outputs = torch.full(
|
||||
(num_tokens, HIDDEN_SIZE), SENTINEL, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
out = moe_fused_mul_sum(
|
||||
inputs=inputs,
|
||||
topk_weights=topk_weights,
|
||||
outputs=outputs,
|
||||
topk_ids=topk_ids,
|
||||
expert_map=expert_map,
|
||||
)
|
||||
|
||||
padding = torch.zeros(num_tokens, dtype=torch.bool, device=device)
|
||||
padding[1] = True
|
||||
assert torch.all(out[1] == SENTINEL), "padding row beside a real row was modified"
|
||||
written = out[~padding].float()
|
||||
assert not written.isnan().any(), "non-local row leaked NaN into the output"
|
||||
torch.testing.assert_close(written, torch.zeros_like(written))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("top_k", TOP_KS)
|
||||
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_weighted_sum_matches_reference(
|
||||
num_tokens: int, top_k: int, hidden_size: int, dtype: torch.dtype
|
||||
):
|
||||
"""Weighted sum over live slots matches the reference across hidden tiles.
|
||||
|
||||
Covers the deployed humming path (globalized ids, expert_map=None): a random
|
||||
-1 mix with NaN-poisoned dead slots, over hidden sizes that span a single
|
||||
tile, a partial trailing tile, and many tiles, in both bf16 and fp32.
|
||||
"""
|
||||
set_random_seed(0)
|
||||
device = "cuda"
|
||||
|
||||
inputs = torch.randn(num_tokens, top_k, hidden_size, dtype=dtype, device=device)
|
||||
topk_weights = torch.rand(num_tokens, top_k, dtype=dtype, device=device)
|
||||
topk_ids = torch.randint(
|
||||
0, NUM_EXPERTS, (num_tokens, top_k), dtype=torch.int32, device=device
|
||||
)
|
||||
invalid = torch.rand(num_tokens, top_k, device=device) < 0.5
|
||||
topk_ids[invalid] = -1
|
||||
inputs[invalid] = float("nan")
|
||||
|
||||
# All-padding rows early-return untouched, so pre-zero to match the reference.
|
||||
outputs = torch.zeros(num_tokens, hidden_size, dtype=dtype, device=device)
|
||||
out = moe_fused_mul_sum(
|
||||
inputs=inputs,
|
||||
topk_weights=topk_weights,
|
||||
outputs=outputs,
|
||||
topk_ids=topk_ids,
|
||||
expert_map=None,
|
||||
)
|
||||
|
||||
assert not out.isnan().any(), "invalid slots leaked into the reduction"
|
||||
ref = _reference(inputs.nan_to_num(), topk_weights, topk_ids, expert_map=None)
|
||||
atol, rtol = (1e-2, 1e-2) if dtype == torch.bfloat16 else (1e-4, 1e-4)
|
||||
torch.testing.assert_close(out.float(), ref, atol=atol, rtol=rtol)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_no_topk_ids_sums_all_slots(dtype: torch.dtype):
|
||||
"""With topk_ids=None every slot is live: plain weighted sum over top_k."""
|
||||
set_random_seed(0)
|
||||
device = "cuda"
|
||||
num_tokens, top_k, hidden_size = 17, 8, 1000
|
||||
|
||||
inputs = torch.randn(num_tokens, top_k, hidden_size, dtype=dtype, device=device)
|
||||
topk_weights = torch.rand(num_tokens, top_k, dtype=dtype, device=device)
|
||||
out = moe_fused_mul_sum(inputs=inputs, topk_weights=topk_weights)
|
||||
|
||||
ref = (inputs.float() * topk_weights.float().unsqueeze(-1)).sum(dim=1)
|
||||
atol, rtol = (1e-2, 1e-2) if dtype == torch.bfloat16 else (1e-4, 1e-4)
|
||||
torch.testing.assert_close(out.float(), ref, atol=atol, rtol=rtol)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hidden_size", [512, 7168])
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_expert_map_is_required_for_nonlocal_ids(hidden_size: int, dtype: torch.dtype):
|
||||
"""expert_map is load-bearing, not redundant: dropping it leaks non-local ids.
|
||||
|
||||
A positive id whose expert is non-local marks a slot the GEMM never wrote, so
|
||||
its `inputs` row is stale. `topk_ids >= 0` alone cannot tell it apart from a
|
||||
real slot -- only expert_map can. This pins the contract the humming path
|
||||
relies on (globalized ids are still routed through expert_map as a safety
|
||||
net): with the map the stale slot is masked; without it, it corrupts the sum.
|
||||
Regression guard against re-dropping expert_map as an optimization.
|
||||
"""
|
||||
set_random_seed(0)
|
||||
device = "cuda"
|
||||
num_tokens, top_k = 8, 8
|
||||
local = NUM_EXPERTS // 2
|
||||
expert_map = _local_expert_map(device)
|
||||
|
||||
inputs = torch.randn(num_tokens, top_k, hidden_size, dtype=dtype, device=device)
|
||||
topk_weights = torch.rand(num_tokens, top_k, dtype=dtype, device=device)
|
||||
# Each row keeps at least one local slot (so the mapped output is finite) and
|
||||
# at least one non-local slot whose stale row is poisoned with NaN.
|
||||
half = top_k // 2
|
||||
topk_ids = torch.empty(num_tokens, top_k, dtype=torch.int32, device=device)
|
||||
topk_ids[:, :half] = torch.randint(
|
||||
0, local, (num_tokens, half), dtype=torch.int32, device=device
|
||||
)
|
||||
topk_ids[:, half:] = torch.randint(
|
||||
local, NUM_EXPERTS, (num_tokens, top_k - half), dtype=torch.int32, device=device
|
||||
)
|
||||
inputs[:, half:] = float("nan")
|
||||
|
||||
atol, rtol = (1e-2, 1e-2) if dtype == torch.bfloat16 else (1e-4, 1e-4)
|
||||
|
||||
mapped = moe_fused_mul_sum(
|
||||
inputs=inputs,
|
||||
topk_weights=topk_weights,
|
||||
outputs=torch.zeros(num_tokens, hidden_size, dtype=dtype, device=device),
|
||||
topk_ids=topk_ids,
|
||||
expert_map=expert_map,
|
||||
)
|
||||
assert not mapped.isnan().any(), "expert_map failed to mask stale non-local slot"
|
||||
ref = _reference(inputs.nan_to_num(), topk_weights, topk_ids, expert_map)
|
||||
torch.testing.assert_close(mapped.float(), ref, atol=atol, rtol=rtol)
|
||||
|
||||
# Same call without the map: the non-local slot is summed and the NaN leaks.
|
||||
unmapped = moe_fused_mul_sum(
|
||||
inputs=inputs,
|
||||
topk_weights=topk_weights,
|
||||
outputs=torch.zeros(num_tokens, hidden_size, dtype=dtype, device=device),
|
||||
topk_ids=topk_ids,
|
||||
expert_map=None,
|
||||
)
|
||||
assert unmapped.isnan().any(), "expert_map is not redundant: dropping it must leak"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_num_valid_tokens_bounds_stale_padding(dtype: torch.dtype):
|
||||
"""Rows past num_valid_tokens are left untouched, even with stale ids/NaN.
|
||||
|
||||
Under cudagraph decode the grid is the static padded token count and the
|
||||
tail rows past num_recv hold stale (non -1) ids and stale NaN in `inputs`
|
||||
from a prior replay -- the -1 mask alone cannot spot them. Bounding on
|
||||
num_valid_tokens must skip the tail before any load so its NaN never reaches
|
||||
the output; without the bound it is summed and a masked finalize can then
|
||||
propagate the NaN. Regression guard for the decode accuracy fix.
|
||||
"""
|
||||
set_random_seed(0)
|
||||
device = "cuda"
|
||||
num_tokens, num_recv, top_k, hidden_size = 17, 10, 8, 512
|
||||
|
||||
inputs = torch.randn(num_tokens, top_k, hidden_size, dtype=dtype, device=device)
|
||||
topk_weights = torch.rand(num_tokens, top_k, dtype=dtype, device=device)
|
||||
# All ids valid (>= 0): the stale tail is indistinguishable by the -1 mask.
|
||||
topk_ids = torch.randint(
|
||||
0, NUM_EXPERTS, (num_tokens, top_k), dtype=torch.int32, device=device
|
||||
)
|
||||
inputs[num_recv:] = float("nan") # stale down_output behind the padding tail
|
||||
num_valid = torch.tensor([num_recv], dtype=torch.int32, device=device)
|
||||
|
||||
atol, rtol = (1e-2, 1e-2) if dtype == torch.bfloat16 else (1e-4, 1e-4)
|
||||
|
||||
bounded = moe_fused_mul_sum(
|
||||
inputs=inputs,
|
||||
topk_weights=topk_weights,
|
||||
outputs=torch.full(
|
||||
(num_tokens, hidden_size), SENTINEL, dtype=dtype, device=device
|
||||
),
|
||||
topk_ids=topk_ids,
|
||||
num_valid_tokens=num_valid,
|
||||
)
|
||||
assert torch.all(bounded[num_recv:] == SENTINEL), (
|
||||
"padding tail past num_recv written"
|
||||
)
|
||||
assert not bounded[:num_recv].isnan().any(), "real rows leaked NaN"
|
||||
ref = _reference(
|
||||
inputs[:num_recv].nan_to_num(),
|
||||
topk_weights[:num_recv],
|
||||
topk_ids[:num_recv],
|
||||
expert_map=None,
|
||||
)
|
||||
torch.testing.assert_close(bounded[:num_recv].float(), ref, atol=atol, rtol=rtol)
|
||||
|
||||
# Without the bound the stale tail is summed and leaks NaN into its rows.
|
||||
unbounded = moe_fused_mul_sum(
|
||||
inputs=inputs,
|
||||
topk_weights=topk_weights,
|
||||
outputs=torch.full(
|
||||
(num_tokens, hidden_size), SENTINEL, dtype=dtype, device=device
|
||||
),
|
||||
topk_ids=topk_ids,
|
||||
)
|
||||
assert unbounded[num_recv:].isnan().any(), "num_valid_tokens bound is load-bearing"
|
||||
@@ -0,0 +1,224 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Correctness tests for the persistent cp.async-pipelined SITU + block-FP8
|
||||
quant kernel (``situ_and_mul_quant``).
|
||||
|
||||
Mirrors ``test_silu_mul_fp8_quant_deep_gemm.py`` in structure: build a reference
|
||||
activation + block-FP8 quant, drive random per-row scale magnitudes, exercise
|
||||
the masked (``valid_rows``) contiguous layout, and compare per valid row.
|
||||
|
||||
The kernel computes SITU in fp32 and converts to fp8 with hardware rounding,
|
||||
which torch cannot reproduce bit-exactly. It is therefore validated against the
|
||||
*ideal* FP8 quantizer of the exact fp64 activation: the difference must stay
|
||||
within the unavoidable FP8 block-quant error (within 1 fp8 code for ~all
|
||||
elements, never off by more than 2).
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.activation import situ_and_mul_quant
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
DEVICE = current_platform.device_type
|
||||
FP8_MAX = torch.finfo(fp8_dtype).max
|
||||
|
||||
# The kernel uses fp8 e4m3 conversion, so it is CUDA-only and needs SM89+
|
||||
# (Ada/Hopper). There is no Triton fallback.
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda() or not current_platform.has_device_capability(89),
|
||||
reason="situ_and_mul_quant persistent kernel requires CUDA SM89+ (fp8 e4m3).",
|
||||
)
|
||||
|
||||
# Kimi-K3 baked SITU params. Only (d=3072, beta=4.0, linear_beta=25.0,
|
||||
# group_size=128, fp16/bf16) routes to the persistent cp.async-pipelined kernel;
|
||||
# anything else falls back to the scalar group kernel.
|
||||
SITU_BETA = 4.0
|
||||
SITU_LINEAR_BETA = 25.0
|
||||
SITU_D = 3072
|
||||
GROUP_SIZE = 128
|
||||
|
||||
# (num_tokens, valid_rows). valid_rows=None => all rows valid. 2048/4096 rows
|
||||
# span multiple grid-stride waves of the persistent grid (GRID_DIM = 132*8).
|
||||
CASES = [
|
||||
(1, None),
|
||||
(17, None),
|
||||
(128, None),
|
||||
(2048, None),
|
||||
(4096, None),
|
||||
(2048, 1500), # DeepEP v2 contiguous padding
|
||||
(2048, 1),
|
||||
(4096, 4095),
|
||||
(512, 0), # whole tensor is padding
|
||||
]
|
||||
|
||||
|
||||
def token_random(num_tokens: int, twod: int, dtype: torch.dtype) -> torch.Tensor:
|
||||
"""Per-row random exponent so groups span many scale magnitudes."""
|
||||
base = torch.randn(num_tokens, twod, dtype=torch.float32, device=DEVICE)
|
||||
exps = torch.randint(1, 13, (num_tokens, 1), device=DEVICE).float()
|
||||
return (base * (2.0**exps)).to(dtype)
|
||||
|
||||
|
||||
def ref_situ_block_fp8_quant(
|
||||
inp: torch.Tensor, beta: float, linear_beta: float, group_size: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Ideal FP8 block-quant of the exact fp64 SITU activation.
|
||||
|
||||
The kernel rounds the activation to ``scalar_t`` before quantizing (it stores
|
||||
``(float)(scalar_t)situ(...)``), so the reference does the same before taking
|
||||
the per-group absmax.
|
||||
"""
|
||||
d = inp.shape[-1] // 2
|
||||
gate, up = inp.double().chunk(2, dim=-1)
|
||||
gate_out = beta * torch.tanh(gate / beta) * torch.sigmoid(gate)
|
||||
up_out = linear_beta * torch.tanh(up / linear_beta) if linear_beta > 0 else up
|
||||
act = (gate_out * up_out).to(inp.dtype).float()
|
||||
|
||||
m = act.shape[0]
|
||||
ng = d // group_size
|
||||
a = act.view(m, ng, group_size)
|
||||
scale = a.abs().amax(dim=2).clamp_min(1e-30) / FP8_MAX
|
||||
q = (a / scale.unsqueeze(2)).clamp(-FP8_MAX, FP8_MAX).to(fp8_dtype)
|
||||
return q.view(m, d), scale
|
||||
|
||||
|
||||
def fp8_code_dist(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
|
||||
"""Signed-magnitude fp8 code distance (valid for e4m3fn and e4m3fnuz)."""
|
||||
|
||||
def ordered(x: torch.Tensor) -> torch.Tensor:
|
||||
x = x.view(torch.uint8).to(torch.int32)
|
||||
return torch.where((x >> 7) == 1, -(x & 0x7F), x & 0x7F)
|
||||
|
||||
return (ordered(a) - ordered(b)).abs()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens,valid", CASES)
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half])
|
||||
@torch.inference_mode()
|
||||
def test_situ_and_mul_quant_pipelined(
|
||||
num_tokens: int, valid: int | None, dtype: torch.dtype
|
||||
) -> None:
|
||||
set_random_seed(42)
|
||||
d = SITU_D
|
||||
inp = token_random(num_tokens, 2 * d, dtype)
|
||||
|
||||
# Sentinels: padding rows must keep their bytes (out) and get scale 1.0.
|
||||
out = torch.full((num_tokens, d), 17.0, device=DEVICE).to(fp8_dtype)
|
||||
scale = torch.full(
|
||||
(num_tokens, d // GROUP_SIZE), -1.0, dtype=torch.float32, device=DEVICE
|
||||
)
|
||||
out_sentinel = out.clone()
|
||||
valid_rows = (
|
||||
None if valid is None else torch.tensor(valid, dtype=torch.int32, device=DEVICE)
|
||||
)
|
||||
|
||||
situ_and_mul_quant(
|
||||
out,
|
||||
scale,
|
||||
inp,
|
||||
beta=SITU_BETA,
|
||||
linear_beta=SITU_LINEAR_BETA,
|
||||
group_size=GROUP_SIZE,
|
||||
valid_rows=valid_rows,
|
||||
)
|
||||
|
||||
# Padding contract: skipped rows keep their bytes; their scale becomes 1.0.
|
||||
if valid is not None and valid < num_tokens:
|
||||
assert torch.equal(scale[valid:], torch.ones_like(scale[valid:]))
|
||||
assert torch.equal(
|
||||
out[valid:].view(torch.uint8), out_sentinel[valid:].view(torch.uint8)
|
||||
)
|
||||
|
||||
rows = num_tokens if valid is None else valid
|
||||
if rows == 0:
|
||||
return
|
||||
|
||||
ref_q, ref_s = ref_situ_block_fp8_quant(
|
||||
inp[:rows], SITU_BETA, SITU_LINEAR_BETA, GROUP_SIZE
|
||||
)
|
||||
|
||||
assert torch.isfinite(scale[:rows]).all()
|
||||
assert not torch.isnan(out[:rows].float()).any()
|
||||
|
||||
# Scales: the kernel's fp32 SITU shifts the per-group absmax only
|
||||
# marginally vs the fp64 reference.
|
||||
torch.testing.assert_close(scale[:rows], ref_s, rtol=3e-2, atol=1e-5)
|
||||
|
||||
# FP8 codes: must land within 1 code of the ideal quantizer for ~all
|
||||
# elements, never off by more than 2.
|
||||
dist = fp8_code_dist(out[:rows], ref_q)
|
||||
assert dist.max().item() <= 2
|
||||
assert (dist <= 1).float().mean().item() >= 0.99
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tokens,topk", [(300, 8), (1, 4), (256, 6)])
|
||||
@torch.inference_mode()
|
||||
def test_situ_and_mul_quant_topk_row_bound(tokens: int, topk: int) -> None:
|
||||
"""valid_rows is a token count; the kernel bounds rows at tokens*topk."""
|
||||
set_random_seed(7)
|
||||
d = SITU_D
|
||||
num_tokens = tokens * topk + 37 # padding tail past the valid rows
|
||||
inp = token_random(num_tokens, 2 * d, torch.bfloat16)
|
||||
|
||||
out = torch.full((num_tokens, d), 17.0, device=DEVICE).to(fp8_dtype)
|
||||
scale = torch.full(
|
||||
(num_tokens, d // GROUP_SIZE), -1.0, dtype=torch.float32, device=DEVICE
|
||||
)
|
||||
out_sentinel = out.clone()
|
||||
valid_rows = torch.tensor(tokens, dtype=torch.int32, device=DEVICE)
|
||||
|
||||
situ_and_mul_quant(
|
||||
out,
|
||||
scale,
|
||||
inp,
|
||||
beta=SITU_BETA,
|
||||
linear_beta=SITU_LINEAR_BETA,
|
||||
group_size=GROUP_SIZE,
|
||||
valid_rows=valid_rows,
|
||||
topk=topk,
|
||||
)
|
||||
|
||||
rows = tokens * topk
|
||||
assert torch.equal(scale[rows:], torch.ones_like(scale[rows:]))
|
||||
assert torch.equal(
|
||||
out[rows:].view(torch.uint8), out_sentinel[rows:].view(torch.uint8)
|
||||
)
|
||||
|
||||
ref_q, ref_s = ref_situ_block_fp8_quant(
|
||||
inp[:rows], SITU_BETA, SITU_LINEAR_BETA, GROUP_SIZE
|
||||
)
|
||||
torch.testing.assert_close(scale[:rows], ref_s, rtol=3e-2, atol=1e-5)
|
||||
assert fp8_code_dist(out[:rows], ref_q).max().item() <= 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half])
|
||||
@torch.inference_mode()
|
||||
def test_situ_and_mul_quant_pipelined_deterministic(dtype: torch.dtype) -> None:
|
||||
"""Two runs on the same input must be byte-identical."""
|
||||
set_random_seed(0)
|
||||
d = SITU_D
|
||||
num_tokens = 2048
|
||||
inp = token_random(num_tokens, 2 * d, dtype)
|
||||
|
||||
def run() -> tuple[torch.Tensor, torch.Tensor]:
|
||||
out = torch.empty(num_tokens, d, dtype=fp8_dtype, device=DEVICE)
|
||||
scale = torch.empty(
|
||||
num_tokens, d // GROUP_SIZE, dtype=torch.float32, device=DEVICE
|
||||
)
|
||||
situ_and_mul_quant(
|
||||
out,
|
||||
scale,
|
||||
inp,
|
||||
beta=SITU_BETA,
|
||||
linear_beta=SITU_LINEAR_BETA,
|
||||
group_size=GROUP_SIZE,
|
||||
)
|
||||
return out, scale
|
||||
|
||||
out1, s1 = run()
|
||||
out2, s2 = run()
|
||||
assert torch.equal(out1.view(torch.uint8), out2.view(torch.uint8))
|
||||
assert torch.equal(s1, s2)
|
||||
@@ -0,0 +1,122 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for Humming MXFP4-weight GEMM paired with a block-FP8 activation.
|
||||
|
||||
The Humming kernel already implements MXFP4 weight (group-32 e8m0) with a
|
||||
grouped FP8 activation via WGMMA software dequant -- see the grouped-fp8
|
||||
`test_mxfp4.py` cases in the humming repo, which run on SM90 (H200). Enabling
|
||||
it in vLLM comes down to two things, both exercised here:
|
||||
|
||||
1. A grouped FP8 activation must classify to a float32-scaled key
|
||||
(kFp8Dynamic128Sym), not a uint8 microscale key. MX microscale
|
||||
activations (group size 32) keep their e8m0 (uint8) scale.
|
||||
2. (kMxfp4Static, kFp8Dynamic128Sym) must be an allowed MoE
|
||||
(weight, activation) pair.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
|
||||
HummingExpertsBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8DynamicTokenSym,
|
||||
kMxfp4Static,
|
||||
kMxfp8Dynamic,
|
||||
)
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
# make_humming_moe_quant_config stores per-sublayer humming LayerConfigs but the
|
||||
# config-plumbing asserts below never read them, so a placeholder satisfies the
|
||||
# required arg without pulling in the humming package.
|
||||
_PLACEHOLDER_HUMMING_CONFIGS = {"w13": None, "w2": None}
|
||||
|
||||
|
||||
def test_mxfp4_weight_with_block_fp8_activation_is_supported():
|
||||
"""MXFP4 weight + block-FP8 (group-128) activation is an allowed pair."""
|
||||
assert HummingExpertsBase._supports_quant_scheme(kMxfp4Static, kFp8Dynamic128Sym)
|
||||
|
||||
|
||||
def test_mxfp4_weight_with_per_token_fp8_activation_still_supported():
|
||||
"""The pre-existing per-token FP8 pairing must keep working."""
|
||||
assert HummingExpertsBase._supports_quant_scheme(kMxfp4Static, kFp8DynamicTokenSym)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not has_humming(), reason="humming is not installed")
|
||||
@pytest.mark.parametrize(
|
||||
"group_size,expected_key",
|
||||
[
|
||||
# per-token FP8 -> float32 per-token scale
|
||||
(0, kFp8DynamicTokenSym),
|
||||
# block FP8 -> float32 group-128 scale (the newly-fixed case)
|
||||
(128, kFp8Dynamic128Sym),
|
||||
# MXFP8 microscale -> e8m0 (uint8) group-32 scale (must be preserved)
|
||||
(32, kMxfp8Dynamic),
|
||||
],
|
||||
)
|
||||
def test_humming_fp8_input_schema_to_quant_key(group_size, expected_key):
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
_humming_input_schema_to_quant_key,
|
||||
)
|
||||
from vllm.utils.humming import HummingInputSchema
|
||||
from vllm.utils.humming import dtypes as humming_dtypes
|
||||
|
||||
schema = HummingInputSchema(
|
||||
a_dtype=humming_dtypes.float8e4m3,
|
||||
input_scale_group_size=group_size,
|
||||
)
|
||||
assert _humming_input_schema_to_quant_key(schema) == expected_key
|
||||
|
||||
|
||||
@pytest.mark.skipif(not has_humming(), reason="humming is not installed")
|
||||
def test_humming_bf16_input_schema_is_unquantized():
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
_humming_input_schema_to_quant_key,
|
||||
)
|
||||
from vllm.utils.humming import HummingInputSchema
|
||||
|
||||
# No a_dtype -> unquantized (bf16/fp16) inputs -> None.
|
||||
assert _humming_input_schema_to_quant_key(HummingInputSchema()) is None
|
||||
|
||||
|
||||
def test_block_fp8_activation_quant_config_is_block_quantized():
|
||||
"""A block-FP8 activation group shape yields a block-quantized MoE config
|
||||
(FP8 dtype, [1, 128] block, not per-act-token). This is what makes the
|
||||
DeepEP prepare/finalize step quantize activations to block FP8 *before* the
|
||||
all-to-all dispatch instead of deferring to Humming -- and what makes
|
||||
HummingExpertsBase.expects_unquantized_inputs return False."""
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
make_humming_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
qc = make_humming_moe_quant_config(
|
||||
quant_dtype=current_platform.fp8_dtype(),
|
||||
weight_dtype="float4e2m1",
|
||||
activation_group_shape=GroupShape(row=1, col=128),
|
||||
humming_configs=_PLACEHOLDER_HUMMING_CONFIGS,
|
||||
)
|
||||
assert qc.is_block_quantized
|
||||
assert qc.block_shape == [1, 128]
|
||||
assert not qc.per_act_token_quant
|
||||
assert qc.quant_dtype == current_platform.fp8_dtype()
|
||||
|
||||
|
||||
def test_default_activation_quant_config_defers_to_humming():
|
||||
"""Without a block activation shape the config stays per-token (the deferred
|
||||
path): Humming quantizes internally, preserving pre-existing behavior for
|
||||
every non-block-FP8 scheme."""
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
make_humming_moe_quant_config,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
qc = make_humming_moe_quant_config(
|
||||
quant_dtype=current_platform.fp8_dtype(),
|
||||
weight_dtype="float4e2m1",
|
||||
humming_configs=_PLACEHOLDER_HUMMING_CONFIGS,
|
||||
)
|
||||
assert not qc.is_block_quantized
|
||||
assert qc.per_act_token_quant
|
||||
Reference in New Issue
Block a user