[Quantization] Add ModelOpt IQ2_XS, IQ2_XXS, and Q8_0 support via b12x (#59604)

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
Signed-off-by: Luke Alonso <lalonso@gmail.com>
Co-authored-by: Andrii Skliar <askliar@nvidia.com>
Co-authored-by: Luke Alonso <lalonso@gmail.com>
This commit is contained in:
Andrii Skliar
2026-10-13 09:06:33 -07:00
committed by GitHub
co-authored by Andrii Skliar Luke Alonso
parent ac71b17517
commit 6849ca696a
15 changed files with 1238 additions and 25 deletions
@@ -21,6 +21,7 @@ from vllm.model_executor.kernels.linear import (
B12xMxFp4LinearKernel,
B12xMxfp8LinearKernel,
B12xNvFp4LinearKernel,
B12xNvFp4W4A16LinearKernel,
B12xTensorFP8ScaledMMLinearKernel,
FP8ScaledMMLinearLayerConfig,
Mxfp8LinearLayerConfig,
@@ -782,7 +783,7 @@ def test_b12x_nvfp4_can_implement_supported_config() -> None:
assert reason is None
def test_b12x_backend_preserves_w4a16_fallback(monkeypatch) -> None:
def test_b12x_backend_selects_w4a16(monkeypatch) -> None:
import vllm.model_executor.kernels.linear as linear_mod
monkeypatch.setattr(linear_mod.current_platform, "_enum", PlatformEnum.CUDA)
@@ -792,17 +793,17 @@ def test_b12x_backend_preserves_w4a16_fallback(monkeypatch) -> None:
monkeypatch.setitem(
linear_mod._POSSIBLE_NVFP4_KERNELS,
PlatformEnum.CUDA,
[B12xNvFp4LinearKernel, MarlinNvFp4LinearKernel],
[B12xNvFp4LinearKernel, B12xNvFp4W4A16LinearKernel, MarlinNvFp4LinearKernel],
)
monkeypatch.setattr(
MarlinNvFp4LinearKernel,
B12xNvFp4W4A16LinearKernel,
"is_supported",
classmethod(lambda cls, compute_capability=None: (True, None)),
)
kernel = init_nvfp4_linear_kernel(use_a16=True)
assert isinstance(kernel, MarlinNvFp4LinearKernel)
assert isinstance(kernel, B12xNvFp4W4A16LinearKernel)
def test_b12x_nvfp4_apply_calls_native_blockscaled_gemm(monkeypatch) -> None:
@@ -859,3 +860,227 @@ def test_b12x_nvfp4_apply_calls_native_blockscaled_gemm(monkeypatch) -> None:
layer.alpha,
)
assert kwargs == {"out_dtype": torch.bfloat16}
def test_b12x_nvfp4_w4a16_uses_weight_only_scale_and_a16_dispatch(monkeypatch):
import vllm.model_executor.kernels.linear.nvfp4.b12x as b12x_mod
calls = []
packed_weight = object()
def pack_weight(weight, scale, *, recipe, global_scale):
assert weight is layer.weight
assert scale is layer.weight_scale
assert recipe == "nvfp4"
assert global_scale is layer.weight_global_scale
return packed_weight
def mm(source, weight, *, required_mode):
assert weight is packed_weight
assert required_mode == "a16"
assert source.dtype == torch.bfloat16
assert source.is_contiguous()
calls.append(source)
return torch.full((source.shape[0], 48), 3.0, dtype=source.dtype)
def reject_activation_quantization(*args, **kwargs):
pytest.fail("W4A16 must not quantize activations")
monkeypatch.setattr(b12x_mod, "scaled_fp4_quant", reject_activation_quantization)
monkeypatch.setattr(
b12x_mod,
"_import_b12x_blockscaled",
lambda: types.SimpleNamespace(pack_weight=pack_weight, mm=mm),
)
monkeypatch.setattr(
b12x_mod,
"_import_b12x_intrinsics",
lambda: types.SimpleNamespace(swizzle_block_scale=lambda scale: scale),
)
layer = torch.nn.Module()
layer.weight = torch.nn.Parameter(
torch.empty((48, 64), dtype=torch.uint8), requires_grad=False
)
layer.weight_scale = torch.nn.Parameter(
torch.empty((48, 8), dtype=torch.float8_e4m3fn), requires_grad=False
)
layer.weight_global_scale = torch.tensor([0.125])
kernel = object.__new__(B12xNvFp4W4A16LinearKernel)
kernel.process_weights_after_loading(layer)
assert layer.b12x_warmup_provider is kernel
kernel.get_b12x_warmup_unit(layer, (1, 8), torch.bfloat16).compile()
assert [source.shape[0] for source in calls] == [1, 8]
x = torch.randn((2, 3, 256), dtype=torch.bfloat16)[..., ::2]
output = kernel.apply_weights(layer, x, torch.ones(48, dtype=torch.bfloat16))
torch.testing.assert_close(calls[-1], x.reshape(6, 128))
torch.testing.assert_close(output, torch.full((2, 3, 48), 4.0, dtype=x.dtype))
with pytest.raises(ValueError, match="requires BF16 input"):
kernel.apply_weights(layer, x.float())
@pytest.mark.parametrize("m,n,k", [(2, 136, 96), (17, 256, 256)])
@torch.inference_mode()
def test_b12x_nvfp4_w4a16_preserves_bf16_activations_under_graph_replay(
monkeypatch, default_vllm_config, m, n, k
) -> None:
import vllm.model_executor.kernels.linear.nvfp4.b12x as b12x_mod
supported, reason = B12xNvFp4W4A16LinearKernel.is_supported()
if not supported:
pytest.skip(reason)
def reject_activation_quantization(*args, **kwargs):
pytest.fail("W4A16 must not quantize activations")
monkeypatch.setattr(b12x_mod, "scaled_fp4_quant", reject_activation_quantization)
torch.manual_seed(17)
codes = torch.randint(0, 16, (n, k), device="cuda")
scales = (2.0 ** torch.randint(-2, 2, (n, k // 16), device="cuda")).to(
torch.float8_e4m3fn
)
layer = torch.nn.Module()
layer.weight = torch.nn.Parameter(
(codes[:, ::2] | codes[:, 1::2] << 4).to(torch.uint8), requires_grad=False
)
layer.weight_scale = torch.nn.Parameter(scales, requires_grad=False)
layer.weight_global_scale = torch.nn.Parameter(
torch.tensor([0.125], device="cuda"), requires_grad=False
)
lut = torch.tensor(
[0, 0.5, 1, 1.5, 2, 3, 4, 6, 0, -0.5, -1, -1.5, -2, -3, -4, -6],
device="cuda",
)
decoded = lut[codes] * scales.float().repeat_interleave(16, dim=1) * 0.125
x_storage = torch.randn(1, m, k * 2, device="cuda", dtype=torch.bfloat16)
x = x_storage[..., ::2]
bias = torch.randn(n, device="cuda", dtype=torch.bfloat16)
kernel = B12xNvFp4W4A16LinearKernel(None)
kernel.process_weights_after_loading(layer)
kernel.get_b12x_warmup_unit(layer, (1, 2, 4, 32), x.dtype).compile()
run = torch.compile(
lambda inputs: kernel.apply_weights(layer, inputs, bias),
fullgraph=True,
dynamic=True,
)
run(x)
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
actual = run(x)
for _ in range(2):
x_storage.normal_()
graph.replay()
expected = (x.double() @ decoded.double().T).bfloat16() + bias
torch.testing.assert_close(actual, expected, atol=0.005, rtol=0.01)
def _block_weights(n, k, codec):
block_size, block_bytes = (
(32, 34) if codec == "q8_0" else (256, 66 if codec == "iq2_xxs" else 74)
)
raw = torch.randint(0, 256, (n, k // block_size, block_bytes), dtype=torch.uint8)
raw[..., :2] = torch.full(
(n, k // block_size, 1), 1 / 128, dtype=torch.float16
).view(torch.uint8)
return raw
@pytest.mark.parametrize("codec", ["iq2_xs", "iq2_xxs", "q8_0"])
@pytest.mark.parametrize("profile_first", [False, True])
@torch.inference_mode()
def test_b12x_block_quant_dense_compiled_graph(
default_vllm_config, codec, profile_first
):
supported, reason = B12xNvFp4W4A16LinearKernel.is_supported()
if not supported:
pytest.skip(reason)
from b12x.testing.iq2_xs_reference import dequantize_blocks
from vllm.model_executor.layers.quantization.modelopt_block_quant import (
ModelOptBlockQuantLinearMethod,
)
default_vllm_config.scheduler_config.max_num_batched_tokens = 32
default_vllm_config.scheduler_config.max_num_scheduled_tokens = 32
default_vllm_config.compilation_config.cudagraph_capture_sizes = [1, 4]
method = ModelOptBlockQuantLinearMethod(codec)
raw = _block_weights(128, 256, codec)
decoded = dequantize_blocks(raw).bfloat16().cuda()
layer = torch.nn.Module()
layer.weight = torch.nn.Parameter(raw.cuda(), requires_grad=False)
method.process_weights_after_loading(layer)
assert layer.b12x_block_plan.prepared is None
if profile_first:
method.apply(layer, torch.zeros(32, 256, device="cuda", dtype=torch.bfloat16))
layer.b12x_warmup_provider.get_b12x_warmup_unit(
layer, (1, 4, 32), torch.bfloat16
).compile()
assert layer.b12x_block_plan.prepared is not None
run = torch.compile(lambda x: method.apply(layer, x), fullgraph=True, dynamic=True)
for rows in (1, 4, 17):
x = torch.randn(rows, 256, device="cuda", dtype=torch.bfloat16)
run(x)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
output = run(x)
for _ in range(2):
x.normal_()
allocated = torch.accelerator.memory_allocated()
graph.replay()
torch.accelerator.synchronize()
assert torch.accelerator.memory_allocated() == allocated
expected = (x.float() @ decoded.float().T).bfloat16()
torch.testing.assert_close(output, expected, rtol=0.015, atol=0.03125)
graph.reset()
@pytest.mark.parametrize("id_dtype", [torch.int32, torch.int64])
@pytest.mark.parametrize("profile_first", [False, True])
@torch.inference_mode()
def test_b12x_q8_embedding_graph(default_vllm_config, id_dtype, profile_first):
supported, reason = B12xNvFp4W4A16LinearKernel.is_supported()
if not supported:
pytest.skip(reason)
from b12x.testing.q8_0_reference import dequantize_blocks
from vllm.model_executor.layers.quantization.modelopt_block_quant import (
ModelOptBlockQuantLinearMethod,
)
default_vllm_config.scheduler_config.max_num_batched_tokens = 32
default_vllm_config.scheduler_config.max_num_scheduled_tokens = 32
default_vllm_config.compilation_config.cudagraph_capture_sizes = [1, 4]
method = ModelOptBlockQuantLinearMethod("q8_0")
method.is_embedding = True
raw = _block_weights(128, 256, "q8_0")
decoded = dequantize_blocks(raw).bfloat16().cuda()
layer = torch.nn.Module()
layer.weight = torch.nn.Parameter(raw.cuda(), requires_grad=False)
method.process_weights_after_loading(layer)
assert all(plan.prepared is None for plan in layer.b12x_embedding_plans.values())
if profile_first:
method.embedding(layer, torch.zeros(32, device="cuda", dtype=id_dtype))
layer.b12x_warmup_provider.get_b12x_warmup_unit(
layer, (1, 4, 32), torch.bfloat16
).compile()
assert all(
plan.prepared is not None for plan in layer.b12x_embedding_plans.values()
)
run = torch.compile(
lambda x: method.embedding(layer, x), fullgraph=True, dynamic=True
)
for count in (7, 9, 70):
ids = torch.arange(count, device="cuda", dtype=id_dtype)
torch._dynamo.mark_dynamic(ids, 0)
torch.testing.assert_close(run(ids), decoded[ids.long()], rtol=0, atol=0)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
output = run(ids)
ids.add_(20)
allocated = torch.accelerator.memory_allocated()
graph.replay()
torch.accelerator.synchronize()
assert torch.accelerator.memory_allocated() == allocated
torch.testing.assert_close(output, decoded[ids.long()], rtol=0, atol=0)
graph.reset()
+54
View File
@@ -25,6 +25,60 @@ def test_b12x_warmup_token_counts_cover_serving_regimes() -> None:
) == (1, 2, 8, 128, 2048)
@pytest.mark.parametrize("is_embedding", [False, True])
def test_b12x_block_quant_warmup_covers_each_layers_plans(monkeypatch, is_embedding):
from vllm.model_executor.layers.quantization.modelopt_block_quant import (
ModelOptBlockQuantLinearMethod,
)
from vllm.model_executor.warmup.b12x_warmup import _collect_warmup_units
method = object.__new__(ModelOptBlockQuantLinearMethod)
method.codec = "q8_0"
method.is_embedding = is_embedding
model = torch.nn.ModuleList([torch.nn.Module(), torch.nn.Module()])
calls = []
monkeypatch.setattr(
method, "apply", lambda layer, x: calls.append((layer, x.shape, x.dtype))
)
monkeypatch.setattr(
method,
"embedding",
lambda layer, ids: calls.append((layer, ids.shape, ids.dtype)),
)
for index, layer in enumerate(model):
layer.b12x_warmup_provider = method
if is_embedding:
layer.weight = torch.nn.Parameter(
torch.empty((8, 8, 34), dtype=torch.uint8), requires_grad=False
)
layer.b12x_embedding_plans = {
dtype: SimpleNamespace(
handle=2 * index + offset, query=SimpleNamespace(max_rows=32)
)
for offset, dtype in enumerate((torch.int32, torch.int64))
}
else:
layer.b12x_block_weight = SimpleNamespace(
in_features=256, values=torch.empty(1)
)
layer.b12x_block_plan = SimpleNamespace(
handle=index, token_counts=(1, 4, 32)
)
units = list(_collect_warmup_units(model, (1, 17, 32), torch.bfloat16))
assert len(units) == 2
for unit in units:
unit.compile()
expected = (
[((32,), torch.int32), ((32,), torch.int64)]
if is_embedding
else [((rows, 256), torch.bfloat16) for rows in (1, 4, 17, 32)]
)
assert calls == [
(layer, shape, dtype) for layer in model for shape, dtype in expected
]
@pytest.mark.parametrize(
("kernel_cls", "module_name", "call_name", "layer", "name"),
[