[ROCm][CI] Add MI355 dense NVFP4 and MoRI kernel mirrors (#58281)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
a9f07d0bc5
commit
153242a314
@@ -1886,7 +1886,7 @@ steps:
|
||||
- vllm/config/
|
||||
- tests/kernels/quantization/test_rocm_skinny_gemms.py
|
||||
commands:
|
||||
- pytest -v -s kernels/quantization --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||
- pytest -v -s kernels/quantization -k "not test_nvfp4_dense_emulation" --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||
|
||||
- label: ":amd: (MI300) DeepEP FP8 MoE Kernels" # TBD
|
||||
timeout_in_minutes: 55
|
||||
@@ -4307,6 +4307,35 @@ steps:
|
||||
commands:
|
||||
- pytest -v -s kernels/moe/test_moe_layer.py
|
||||
|
||||
- label: ":amd: (MI355) MoRI EP Numerics"
|
||||
timeout_in_minutes: 20
|
||||
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
|
||||
dind: false
|
||||
agent_pool: mi355_2
|
||||
num_gpus: 2
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
- tests/kernels/moe/test_mori_moe.py
|
||||
- tests/kernels/moe/utils.py
|
||||
- tests/kernels/moe/modular_kernel_tools/parallel_utils.py
|
||||
- tests/kernels/utils.py
|
||||
- tests/utils.py
|
||||
- tests/conftest.py
|
||||
- vllm/utils/torch_utils.py
|
||||
- vllm/model_executor/layers/fused_moe/
|
||||
- vllm/model_executor/layers/quantization/
|
||||
- vllm/distributed/
|
||||
- vllm/config/
|
||||
- vllm/forward_context.py
|
||||
- vllm/v1/worker/workspace.py
|
||||
- vllm/platforms/
|
||||
- vllm/_aiter_ops.py
|
||||
- requirements/rocm-test.txt
|
||||
optional: true
|
||||
commands:
|
||||
- amd-smi
|
||||
- VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_MOE=1 VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0 pytest -v -s kernels/moe/test_mori_moe.py
|
||||
|
||||
- label: ":amd: (MI355) Quantization Kernels Shard %N"
|
||||
timeout_in_minutes: 120
|
||||
parallelism: 6
|
||||
@@ -4332,7 +4361,30 @@ steps:
|
||||
- tests/kernels/quantization/test_rocm_skinny_gemms.py
|
||||
optional: true
|
||||
commands:
|
||||
- pytest -v -s kernels/quantization --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||
- pytest -v -s kernels/quantization -k "not test_nvfp4_dense_emulation" --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||
|
||||
- label: ":amd: (MI355) NVFP4 Dense Emulation Shard %N"
|
||||
timeout_in_minutes: 10
|
||||
parallelism: 3
|
||||
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
|
||||
dind: false
|
||||
agent_pool: mi355_1
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
- tests/kernels/quantization/test_nvfp4_emulation.py
|
||||
- tests/kernels/quantization/nvfp4_utils.py
|
||||
- tests/conftest.py
|
||||
- vllm/model_executor/kernels/linear/
|
||||
- vllm/model_executor/layers/quantization/utils/nvfp4_emulation_utils.py
|
||||
- vllm/config/kernel.py
|
||||
- vllm/platforms/
|
||||
- vllm/envs.py
|
||||
- vllm/utils/torch_utils.py
|
||||
- requirements/rocm-test.txt
|
||||
optional: true
|
||||
commands:
|
||||
- amd-smi
|
||||
- pytest -v -s kernels/quantization/test_nvfp4_emulation.py -k test_nvfp4_dense_emulation --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||
|
||||
- label: ":amd: (MI355) FP8 MoE Kernels" # TBD
|
||||
timeout_in_minutes: 40
|
||||
|
||||
@@ -330,6 +330,8 @@ steps:
|
||||
- vllm/envs.py
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
commands:
|
||||
- pytest -v -s kernels/quantization -k "not test_nvfp4_dense_emulation" --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||
|
||||
- label: ":nvidia: (L4) MoE Kernels Shard %N"
|
||||
key: kernels-moe-test
|
||||
@@ -522,6 +524,32 @@ steps:
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355) NVFP4 Dense Emulation Shard %N"
|
||||
dind: false
|
||||
device: mi355_1
|
||||
num_devices: 1
|
||||
timeout_in_minutes: 10
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- tests/kernels/quantization/test_nvfp4_emulation.py
|
||||
- tests/kernels/quantization/nvfp4_utils.py
|
||||
- tests/conftest.py
|
||||
- vllm/model_executor/kernels/linear/
|
||||
- vllm/model_executor/layers/quantization/utils/nvfp4_emulation_utils.py
|
||||
- vllm/config/kernel.py
|
||||
- vllm/platforms/
|
||||
- vllm/envs.py
|
||||
- vllm/utils/torch_utils.py
|
||||
- requirements/rocm-test.txt
|
||||
# W4A4 format numerics and replay via emulation. TODO: native packed
|
||||
# activation/KV-cache layouts and CUDA-only kernels need AMD backends.
|
||||
commands:
|
||||
- amd-smi
|
||||
- pytest -v -s kernels/quantization/test_nvfp4_emulation.py -k test_nvfp4_dense_emulation --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||
|
||||
- label: ":nvidia: (DGX) Spark b12x Linear Kernels Nightly"
|
||||
key: b12x-linear-kernels-dgx-spark-nightly
|
||||
@@ -701,6 +729,38 @@ steps:
|
||||
commands:
|
||||
- pytest -v -s kernels/moe/test_moe_layer.py
|
||||
- pytest -v -s kernels/moe/test_deepep_v2_moe.py
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355) MoRI EP Numerics"
|
||||
dind: false
|
||||
device: mi355_2
|
||||
num_devices: 2
|
||||
timeout_in_minutes: 20
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- tests/kernels/moe/test_mori_moe.py
|
||||
- tests/kernels/moe/utils.py
|
||||
- tests/kernels/moe/modular_kernel_tools/parallel_utils.py
|
||||
- tests/kernels/utils.py
|
||||
- tests/utils.py
|
||||
- tests/conftest.py
|
||||
- vllm/utils/torch_utils.py
|
||||
- vllm/model_executor/layers/fused_moe/
|
||||
- vllm/model_executor/layers/quantization/
|
||||
- vllm/distributed/
|
||||
- vllm/config/
|
||||
- vllm/forward_context.py
|
||||
- vllm/v1/worker/workspace.py
|
||||
- vllm/platforms/
|
||||
- vllm/_aiter_ops.py
|
||||
- requirements/rocm-test.txt
|
||||
# Same-node MoRI/AITER BF16 and per-token/channel FP8. TODO: the
|
||||
# DeepEP v2 FlashInfer backend matrix has no direct AMD implementation.
|
||||
commands:
|
||||
- amd-smi
|
||||
- VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_MOE=1 VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0 pytest -v -s kernels/moe/test_mori_moe.py
|
||||
|
||||
- label: ":amd: (MI355) FP8 MoE Kernels"
|
||||
key: amd-fp8-moe-kernels-mi355
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Two-rank MoRI/AITER numerical counterpart to the DeepEP v2 MoE matrix."""
|
||||
|
||||
from dataclasses import replace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.moe.modular_kernel_tools.parallel_utils import (
|
||||
parallel_launch_with_config,
|
||||
)
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config, make_test_weights
|
||||
from tests.kernels.utils import torch_experts
|
||||
from tests.utils import multi_gpu_test
|
||||
from vllm.config import ParallelConfig, SchedulerConfig, VllmConfig
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
maybe_make_prepare_finalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_mori
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
MNKS = [
|
||||
(1, 256, 256),
|
||||
(2, 256, 512),
|
||||
(3, 1024, 2048),
|
||||
(32, 256, 1024),
|
||||
(45, 512, 2048),
|
||||
(64, 1024, 1024),
|
||||
(222, 1024, 2048),
|
||||
]
|
||||
|
||||
|
||||
def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, fp8, graph):
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import AiterExperts
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.mori import (
|
||||
MoriPrepareAndFinalize,
|
||||
)
|
||||
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
init_workspace_manager(pgi.device)
|
||||
set_random_seed(47)
|
||||
dtype = current_platform.fp8_dtype() if fp8 else None
|
||||
(_, w1, s1, _), (_, w2, s2, _) = make_test_weights(
|
||||
32, n, k, quant_dtype=dtype, per_out_ch_quant=fp8
|
||||
)
|
||||
# Share global expert weights, but give each rank different token inputs.
|
||||
set_random_seed(101 + pgi.rank)
|
||||
x = torch.randn(m, k, device=pgi.device, dtype=torch.bfloat16) / 10
|
||||
routing = torch.randn(m, 32, device=pgi.device)
|
||||
# Guarantee a route to both ranks, including the one-token case.
|
||||
routing[:, 0] = routing[:, 16] = float("inf")
|
||||
ids = routing.topk(6, dim=-1).indices.to(torch.int32)
|
||||
weights = torch.randn(m, 6, device=pgi.device, dtype=torch.float32)
|
||||
parallel = replace(
|
||||
FusedMoEParallelConfig.make_no_parallel(),
|
||||
dp_size=2,
|
||||
dp_rank=pgi.rank,
|
||||
ep_size=2,
|
||||
ep_rank=pgi.rank,
|
||||
use_ep=True,
|
||||
all2all_backend="mori_high_throughput",
|
||||
)
|
||||
config = replace(
|
||||
make_dummy_moe_config(
|
||||
num_experts=32,
|
||||
num_local_experts=16,
|
||||
experts_per_token=6,
|
||||
hidden_dim=k,
|
||||
intermediate_size=n,
|
||||
max_num_tokens=256,
|
||||
),
|
||||
moe_parallel_config=parallel,
|
||||
)
|
||||
start, end = pgi.rank * 16, (pgi.rank + 1) * 16
|
||||
quant = FusedMoEQuantConfig.make(
|
||||
dtype,
|
||||
per_act_token_quant=fp8,
|
||||
per_out_ch_quant=fp8,
|
||||
w1_scale=None if s1 is None else s1[start:end].contiguous(),
|
||||
w2_scale=None if s2 is None else s2[start:end].contiguous(),
|
||||
)
|
||||
prepare = maybe_make_prepare_finalize(config, quant)
|
||||
assert isinstance(prepare, MoriPrepareAndFinalize)
|
||||
assert prepare.use_fp8_dispatch == fp8
|
||||
assert prepare.num_dispatchers() == 2
|
||||
experts = AiterExperts(config, quant)
|
||||
assert not experts.expects_unquantized_inputs
|
||||
kernel = FusedMoEKernel(prepare, experts)
|
||||
local_w1, local_w2 = rocm_aiter_ops.shuffle_weights(
|
||||
w1[start:end].contiguous(), w2[start:end].contiguous()
|
||||
)
|
||||
local_w1.is_shuffled = local_w2.is_shuffled = True
|
||||
expert_mask = torch.zeros(32, dtype=torch.int32, device=pgi.device)
|
||||
expert_mask[start:end] = 1
|
||||
|
||||
def reference(route_weights):
|
||||
return torch_experts(
|
||||
x,
|
||||
w1,
|
||||
w2,
|
||||
route_weights,
|
||||
ids,
|
||||
w1_scale=s1,
|
||||
w2_scale=s2,
|
||||
quant_dtype=dtype,
|
||||
per_act_token_quant=fp8,
|
||||
)
|
||||
|
||||
def run():
|
||||
with set_forward_context(None, vllm_config):
|
||||
return kernel.apply(
|
||||
x,
|
||||
local_w1,
|
||||
local_w2,
|
||||
weights,
|
||||
ids,
|
||||
activation=MoEActivation.SILU,
|
||||
global_num_experts=32,
|
||||
expert_map=expert_mask,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
def check(actual):
|
||||
expected = reference(weights)
|
||||
assert actual.shape == expected.shape
|
||||
assert actual.dtype == expected.dtype
|
||||
local_weights = weights * ((ids >= start) & (ids < end))
|
||||
missing_remote = reference(local_weights)
|
||||
norm = expected.float().norm()
|
||||
assert norm > 0
|
||||
norm_budget = 0.2 if fp8 else 0.06
|
||||
assert (actual.float() - expected.float()).norm() / norm < norm_budget
|
||||
# The norm check also rejects low-signal zero or rank-local results.
|
||||
for incorrect in (torch.zeros_like(expected), missing_remote):
|
||||
assert (incorrect.float() - expected.float()).norm() / norm > norm_budget
|
||||
if fp8:
|
||||
assert (
|
||||
torch.isclose(actual, expected, atol=0.2, rtol=0.2)
|
||||
.float()
|
||||
.mean()
|
||||
.item()
|
||||
> 0.99
|
||||
)
|
||||
else:
|
||||
torch.testing.assert_close(actual, expected, atol=0.06, rtol=0.06)
|
||||
assert torch.isfinite(actual).all()
|
||||
|
||||
actual = run()
|
||||
check(actual)
|
||||
if graph:
|
||||
saved = actual.clone()
|
||||
run()
|
||||
capture = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(capture):
|
||||
captured = run()
|
||||
for factor in (-0.5, 1.5):
|
||||
x.mul_(factor)
|
||||
weights.mul_(-0.75)
|
||||
capture.replay()
|
||||
torch.accelerator.synchronize()
|
||||
check(captured)
|
||||
torch.testing.assert_close(actual, saved, atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm() or not has_mori(), reason="Requires ROCm MoRI"
|
||||
)
|
||||
@pytest.mark.parametrize("m,n,k", MNKS)
|
||||
@pytest.mark.parametrize("fp8", [False, True], ids=["bf16", "fp8_token_channel"])
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_mori_moe(m, n, k, fp8):
|
||||
config = VllmConfig(
|
||||
parallel_config=ParallelConfig(
|
||||
data_parallel_size=2,
|
||||
enable_expert_parallel=True,
|
||||
all2all_backend="mori_high_throughput",
|
||||
),
|
||||
scheduler_config=SchedulerConfig.default_factory(max_num_batched_tokens=256),
|
||||
)
|
||||
env = {
|
||||
"VLLM_ROCM_USE_AITER": "1",
|
||||
"VLLM_ROCM_USE_AITER_MOE": "1",
|
||||
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "0",
|
||||
}
|
||||
parallel_launch_with_config(2, _mori_worker, config, env, m, n, k, fp8, m == 32)
|
||||
@@ -4,9 +4,21 @@ from typing import cast
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from nvfp4_utils import (
|
||||
FLOAT4_E2M1_MAX,
|
||||
FLOAT8_E4M3_MAX,
|
||||
break_fp4_bytes,
|
||||
kE2M1ToFloat,
|
||||
)
|
||||
from safetensors import safe_open
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.config.kernel import KernelConfig
|
||||
from vllm.model_executor.kernels.linear import init_nvfp4_linear_kernel
|
||||
from vllm.model_executor.kernels.linear.nvfp4.emulation import (
|
||||
EmulationNvFp4LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
@@ -34,6 +46,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
from vllm.triton_utils import triton
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
@@ -769,3 +782,85 @@ def test_nvfp4_moe_correctness(
|
||||
atol=0.0 if on_gfx950() else 0.02,
|
||||
rtol=0,
|
||||
)
|
||||
|
||||
|
||||
def _dense_nvfp4_reference(
|
||||
layer: torch.nn.Module, x: torch.Tensor, bias: torch.Tensor | None
|
||||
) -> torch.Tensor:
|
||||
"""Decode E2M1 nibbles and both scale levels without production helpers."""
|
||||
levels = kE2M1ToFloat.cpu()
|
||||
weight = break_fp4_bytes(layer.weight.cpu(), torch.float32)
|
||||
weight *= (
|
||||
layer.weight_scale.cpu().float() * layer.weight_global_scale.cpu()
|
||||
).repeat_interleave(16, dim=-1)
|
||||
blocks = x.cpu().float().reshape(x.shape[0], -1, 16)
|
||||
global_scale = layer.input_global_scale_inv.cpu()
|
||||
scales = (
|
||||
(blocks.abs().amax(-1, keepdim=True) * (1.0 / FLOAT4_E2M1_MAX) * global_scale)
|
||||
.clamp(max=FLOAT8_E4M3_MAX)
|
||||
.to(torch.float8_e4m3fn)
|
||||
.float()
|
||||
)
|
||||
inverse = torch.where(scales == 0, 0, global_scale / scales)
|
||||
scaled = (blocks * inverse).abs().clamp(max=FLOAT4_E2M1_MAX)
|
||||
distances = (scaled.unsqueeze(-1) - levels).abs()
|
||||
# E2M1 ties round to the even significand (even encoding).
|
||||
nearest_mask = distances == distances.amin(-1, keepdim=True)
|
||||
even_mask = nearest_mask & (torch.arange(8, device="cpu") % 2 == 0)
|
||||
candidates = torch.where(even_mask.any(-1, keepdim=True), even_mask, nearest_mask)
|
||||
nearest = candidates.to(torch.int8).argmax(-1)
|
||||
activation = levels[nearest] * blocks.sign() * (scales / global_scale)
|
||||
expected = activation.reshape_as(x).to(x.dtype) @ weight.to(x.dtype).T
|
||||
if bias is not None:
|
||||
expected += bias.cpu()
|
||||
return expected
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm NVFP4 fallback")
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize("m,n,k", [(1, 32, 64), (17, 128, 256)])
|
||||
@pytest.mark.parametrize("with_bias", [False, True])
|
||||
def test_nvfp4_dense_emulation(
|
||||
dtype: torch.dtype, m: int, n: int, k: int, with_bias: bool
|
||||
) -> None:
|
||||
"""W4A4 fallback preserves packed scales, bias and outputs across replay."""
|
||||
set_random_seed(47)
|
||||
layer = torch.nn.Module()
|
||||
# Every signed E2M1 code appears, including both zero encodings.
|
||||
codes = (torch.randperm(n * k, device="cuda") % 16).reshape(n, k)
|
||||
layer.weight = (codes[:, ::2] | (codes[:, 1::2] << 4)).to(torch.uint8)
|
||||
layer.weight_scale = (
|
||||
torch.arange(n * k // 16, device="cuda").reshape(n, k // 16) % 7 / 8
|
||||
).to(torch.float8_e4m3fn)
|
||||
layer.weight_global_scale = torch.tensor(0.25, device="cuda")
|
||||
layer.input_global_scale_inv = torch.tensor(3.0, device="cuda")
|
||||
x = torch.randn(m, k, device="cuda", dtype=dtype)
|
||||
x[:, :16] = 0
|
||||
bias = torch.randn(n, device="cuda", dtype=dtype) if with_bias else None
|
||||
with set_current_vllm_config(
|
||||
VllmConfig(kernel_config=KernelConfig(linear_backend="emulation"))
|
||||
):
|
||||
kernel = init_nvfp4_linear_kernel()
|
||||
assert isinstance(kernel, EmulationNvFp4LinearKernel)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
original_weight = layer.weight.clone()
|
||||
original_scale = layer.weight_scale.view(torch.uint8).clone()
|
||||
actual = kernel.apply_weights(layer, x, bias)
|
||||
torch.testing.assert_close(
|
||||
actual.cpu(), _dense_nvfp4_reference(layer, x, bias), atol=0.03, rtol=0.01
|
||||
)
|
||||
assert actual.abs().max() > 0.1
|
||||
saved = actual.clone()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
captured = kernel.apply_weights(layer, x, bias)
|
||||
for sign in (-1, 1):
|
||||
x.mul_(sign * 0.75)
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
torch.testing.assert_close(
|
||||
captured.cpu(), _dense_nvfp4_reference(layer, x, bias), atol=0.03, rtol=0.01
|
||||
)
|
||||
torch.testing.assert_close(actual, saved, atol=0, rtol=0)
|
||||
assert torch.equal(layer.weight, original_weight)
|
||||
assert torch.equal(layer.weight_scale.view(torch.uint8), original_scale)
|
||||
|
||||
Reference in New Issue
Block a user