[ROCm][Kimi-K3] Fuse AttnRes output with per-token FP8 input quantization (#59069)

Signed-off-by: whx-sjtu <xiaowang990929@gmail.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Fangzhou Ai <31551580+Fangzhou-Ai@users.noreply.github.com>
This commit is contained in:
Hexiang Wang
2026-10-08 12:01:47 +00:00
committed by GitHub
co-authored by OpenAI Codex Fangzhou Ai
parent 73c742b09c
commit 458ba2edf8
7 changed files with 455 additions and 15 deletions
@@ -0,0 +1,165 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Eight-GPU AttnRes+FP8 projection comparison with rotated graph timing."""
import argparse
import json
import os
import statistics
from pathlib import Path
import torch
import torch.distributed as dist
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.models.kimi_k3.amd.ops.attn_res import attn_res
from vllm.platforms import current_platform
def capture(fn, repeats):
stream = torch.cuda.Stream()
stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(stream):
for _ in range(5):
fn()
torch.cuda.current_stream().wait_stream(stream)
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
for _ in range(repeats):
fn()
return graph
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument(
"--tokens", type=int, nargs="+", default=[1, 4, 16, 64, 256, 1024, 4096, 16384]
)
parser.add_argument("--blocks", type=int, nargs="+", default=[1, 4, 8])
args = parser.parse_args()
assert int(os.environ["WORLD_SIZE"]) == 8
assert torch.accelerator.device_count() == 8
rank = int(os.environ["LOCAL_RANK"])
torch.accelerator.set_device_index(rank)
torch.set_num_threads(1)
torch.manual_seed(1729 + rank)
dist.init_process_group("gloo")
args.output.mkdir(parents=True, exist_ok=True)
dtype = current_platform.fp8_dtype()
rows = []
with set_current_vllm_config(VllmConfig()):
weight = torch.randn(2112, 7168, device="cuda", dtype=torch.bfloat16) * 0.02
weight, ws = rocm_aiter_ops.per_token_quant(weight, dtype)
weight = rocm_aiter_ops.shuffle_weight(weight)
for m in args.tokens:
prefix = torch.randn(m, 7168, device="cuda", dtype=torch.bfloat16)
delta = torch.zeros_like(prefix)
blocks = torch.randn(m, 9, 7168, device="cuda", dtype=torch.bfloat16)
norm = torch.ones(7168, device="cuda", dtype=torch.bfloat16)
score = torch.randn_like(norm) / 7168**0.5
for b in args.blocks:
def produce(
fused,
prefix=prefix,
delta=delta,
blocks=blocks,
norm=norm,
score=score,
b=b,
):
result = attn_res(
prefix,
delta,
blocks,
norm,
score,
norm,
b,
-1,
1e-5,
1e-5,
quant_dtype=dtype if fused else None,
)
return (
result
if fused
else rocm_aiter_ops.per_token_quant(result, dtype)
)
def segment(fused, produce=produce):
x, scale = produce(fused)
return rocm_aiter_ops.preshuffled_per_token_w8a8_gemm(
x, weight, scale, ws, output_dtype=torch.bfloat16
)
expected, actual = segment(False), segment(True)
relative_l2 = (
(actual.float() - expected.float()).norm() / expected.float().norm()
).item()
assert relative_l2 < 0.002, relative_l2
repeats = 20 if m < 1024 else 4
functions = {
"baseline_producer": lambda produce=produce: produce(False),
"fused_producer": lambda produce=produce: produce(True),
"baseline_segment": lambda segment=segment: segment(False),
"fused_segment": lambda segment=segment: segment(True),
}
graphs = {k: capture(fn, repeats) for k, fn in functions.items()}
samples = {k: [] for k in functions}
names = list(functions)
for sample in range(9):
for name in names[sample % 4 :] + names[: sample % 4]:
dist.barrier()
graphs[name].replay()
torch.accelerator.synchronize()
start, end = [
torch.cuda.Event(enable_timing=True) for _ in range(2)
]
start.record()
graphs[name].replay()
end.record()
end.synchronize()
samples[name].append(start.elapsed_time(end) * 1000 / repeats)
row = {
"tokens": m,
"blocks": b,
"relative_l2": relative_l2,
"samples_us": samples,
"median_us": {k: statistics.median(v) for k, v in samples.items()},
}
rows.append(row)
print(
json.dumps(
{
"rank": rank,
"tokens": m,
"blocks": b,
"median_us": row["median_us"],
}
),
flush=True,
)
del graphs
(args.output / f"rank-{rank}.json").write_text(
json.dumps(
{
"rank": rank,
"world_size": 8,
"torch": torch.__version__,
"rows": rows,
"scope": "AttnRes + FP8 quant + TP8 MLA input projection",
},
indent=2,
)
+ "\n"
)
dist.barrier()
dist.destroy_process_group()
if __name__ == "__main__":
main()
+99
View File
@@ -191,3 +191,102 @@ def test_amd_attn_res_fused_contract(
original_blocks[:, block_write_idx].copy_(expected_prefix)
torch.testing.assert_close(blocks, original_blocks, atol=0, rtol=0)
assert actual.is_contiguous()
@pytest.mark.parametrize(
"num_tokens,num_blocks,has_delta,write_block",
[
(0, 0, False, False),
(1, 0, True, True),
(17, 4, True, True),
(320, 8, False, False),
],
)
def test_amd_attn_res_fp8_preserves_prefix_and_quantized_output(
num_tokens,
num_blocks,
has_delta,
write_block,
default_vllm_config,
):
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
torch.manual_seed(41)
hidden_size = 7168
prefix = _randn_with_row_padding(num_tokens, hidden_size, padding=7)
delta = _randn_with_row_padding(num_tokens, hidden_size) if has_delta else None
blocks = _randn_with_row_padding(num_tokens, 9, hidden_size, padding=13)
norm = 1 + torch.randn(hidden_size, device="cuda", dtype=torch.bfloat16) * 0.1
score = torch.randn_like(norm) / hidden_size**0.5
out_norm = 1 + torch.randn_like(norm) * 0.1
ref_prefix, ref_blocks = prefix.clone(), blocks.clone()
kwargs = dict(
num_blocks=num_blocks,
block_write_idx=num_blocks if write_block else -1,
eps=1e-5,
output_norm_eps=1e-5,
)
reference = attn_res(ref_prefix, delta, ref_blocks, norm, score, out_norm, **kwargs)
output, scale = attn_res(
prefix,
delta,
blocks,
norm,
score,
out_norm,
quant_dtype=current_platform.fp8_dtype(),
**kwargs,
)
assert output.dtype == current_platform.fp8_dtype()
assert scale.shape == (num_tokens, 1)
if num_tokens:
# Keep reference rounding independent of Inductor's fused quantization.
quant = QuantFP8(
static=False, group_shape=GroupShape.PER_TOKEN, compile_native=False
)
ref_output, ref_scale = quant(reference)
torch.testing.assert_close(scale, ref_scale, atol=1e-7, rtol=1e-6)
# Floating-point fusion can move values at FP8 rounding midpoints.
# Bound each change to the adjacent code and bound how often it occurs.
code_delta = (
output.view(torch.uint8).to(torch.int16)
- ref_output.view(torch.uint8).to(torch.int16)
).abs()
assert code_delta.max().item() <= 1
assert (code_delta != 0).float().mean().item() < 1e-4
torch.testing.assert_close(prefix, ref_prefix, atol=0, rtol=0)
torch.testing.assert_close(blocks, ref_blocks, atol=0, rtol=0)
@pytest.mark.parametrize(
"matching,requires_unquantized", [(True, False), (False, False), (True, True)]
)
def test_mla_input_quant_requires_all_consumers(matching, requires_unquantized):
from types import SimpleNamespace
from torch import nn
from vllm.model_executor.layers.fusion.quant_activation import (
expose_input_quant_key,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8DynamicTokenSym,
)
from vllm.models.kimi_k3.amd.mla import KimiK3MultiHeadLatentAttentionWrapper
attention = object.__new__(KimiK3MultiHeadLatentAttentionWrapper)
nn.Module.__init__(attention)
attention.indexer = None
attention.q_lora_rank = 1536
attention.fused_qkv_a_proj = nn.Identity()
attention.g_proj = nn.Identity()
kernel = SimpleNamespace(input_quant_key=lambda: kFp8DynamicTokenSym)
expose_input_quant_key(attention.fused_qkv_a_proj, kernel)
if matching:
expose_input_quant_key(attention.g_proj, kernel)
attention.g_proj.requires_unquantized_input = requires_unquantized
actual = attention.get_input_quant_key()
assert actual == (
kFp8DynamicTokenSym if matching and not requires_unquantized else None
)
@@ -1,6 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from torch import nn
@@ -36,3 +39,81 @@ def test_mla_self_attention_returns_projection_storage_directly() -> None:
assert output is projected
assert output.data_ptr() == projected.data_ptr()
@pytest.mark.parametrize("quantized", [False, True])
@pytest.mark.parametrize("fused_decode", [False, True])
def test_mla_forward_preserves_output_shape_with_fp8_input(
monkeypatch, quantized: bool, fused_decode: bool
) -> None:
"""Both MLA paths accept producer-quantized input after decode fusion."""
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8DynamicTokenSym,
)
from vllm.models.kimi_k3.amd.mla import KimiK3MultiHeadLatentAttentionWrapper
hidden_states = torch.randn(4, 8, dtype=torch.bfloat16)
if quantized:
hidden_states = QuantizedActivation(
hidden_states.to(kFp8DynamicTokenSym.dtype),
torch.ones(4, 1),
hidden_states.dtype,
hidden_states.shape,
kFp8DynamicTokenSym,
)
projected = torch.randn(4, 8, dtype=torch.bfloat16)
paths = []
def input_projection(value):
assert value is hidden_states
return torch.zeros(4, 8, dtype=torch.bfloat16), None
def attention_output(path, output_shape):
assert output_shape == projected.shape
paths.append(path)
return projected
attention = SimpleNamespace(
q_lora_rank=2,
kv_lora_rank=4,
qk_rope_head_dim=2,
qk_head_dim=4,
num_heads=2,
v_head_dim=4,
fused_qkv_a_proj=input_projection,
q_a_layernorm=nn.Identity(),
_normalize_q_kv=lambda q, kv: (q, kv),
q_b_proj=lambda q: (torch.zeros(4, 8, dtype=q.dtype), None),
rotary_emb=None,
indexer=None,
dcp_q_replicate=False,
_fused_qk_prep=fused_decode,
_fused_decode=lambda q, kv, pe, pos, shape, *args: attention_output(
"fused", shape
),
g_proj=None,
o_proj=lambda value: (value, None),
)
if fused_decode:
attention.mla_attn = SimpleNamespace(layer_name="test")
monkeypatch.setattr(
"vllm.models.kimi_k3.amd.mla.get_attention_context",
lambda _: (
SimpleNamespace(num_actual_tokens=4, num_decode_tokens=4),
None,
torch.zeros(1),
torch.arange(4),
),
)
else:
attention.mla_attn = lambda q, kv, pe, **kwargs: attention_output(
"regular", kwargs["output_shape"]
)
output = KimiK3MultiHeadLatentAttentionWrapper.forward(
attention, torch.arange(4), hidden_states
)
assert output is projected
assert paths == ["fused" if fused_decode else "regular"]
+12 -4
View File
@@ -13,6 +13,10 @@ from vllm.config import VllmConfig
from vllm.distributed import divide
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.layers.fusion.quant_activation import (
QuantizedActivation,
get_input_quant_key,
)
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
RowParallelLinear,
@@ -37,6 +41,7 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
causal_conv1d_fn,
causal_conv1d_update,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
from vllm.model_executor.utils import set_weight_attrs
from vllm.models.kimi_k3.amd.kda_metadata import KimiK3ROCmKDABackend
@@ -300,13 +305,16 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
qkv = qkv.permute(1, 0, 2, 3).contiguous().unsqueeze(1)
return qkv.unbind(0)
def get_input_quant_key(self) -> QuantKey | None:
return get_input_quant_key(self.in_proj_qkvgfab)
def forward(
self,
hidden_states: torch.Tensor,
hidden_states: torch.Tensor | QuantizedActivation,
positions: torch.Tensor,
) -> torch.Tensor:
num_tokens = hidden_states.size(0)
projected_qkvgfab = self.in_proj_qkvgfab(hidden_states)[0]
num_tokens = projected_qkvgfab.shape[0]
split_sizes = [
3 * self.local_projection_size,
@@ -326,8 +334,8 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
core_attn_out = torch.empty(
(1, num_tokens, self.local_num_heads, self.head_dim),
dtype=hidden_states.dtype,
device=hidden_states.device,
dtype=projected_qkvgfab.dtype,
device=projected_qkvgfab.device,
)
self._forward(
+44 -5
View File
@@ -2,11 +2,12 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
from typing import Any
from typing import Any, cast
import torch
from torch import nn
from vllm import envs
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import (
get_pp_group,
@@ -19,6 +20,10 @@ from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
from vllm.model_executor.layers.fusion.quant_activation import (
QuantizedActivation,
get_input_quant_key,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -38,6 +43,10 @@ from vllm.model_executor.layers.mamba.mamba_utils import (
)
from vllm.model_executor.layers.mla import MLAModules
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8DynamicTokenSym,
)
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
@@ -114,6 +123,9 @@ class KimiMLP(nn.Module):
"Only silu and situ are supported."
)
def get_input_quant_key(self) -> QuantKey | None:
return get_input_quant_key(self.gate_up_proj)
def forward(self, x):
gate_up, _ = self.gate_up_proj(x)
x = self.act_fn(gate_up)
@@ -148,8 +160,10 @@ def _apply_attn_res(
delta: torch.Tensor | None = None,
output_norm: RMSNorm | None = None,
block_write_idx: int = -1,
) -> torch.Tensor:
return attn_res(
quant_key: QuantKey | None = None,
) -> torch.Tensor | QuantizedActivation:
fuse_quant = quant_key == kFp8DynamicTokenSym and not envs.VLLM_BATCH_INVARIANT
result = attn_res(
prefix_sum,
delta,
block_residual,
@@ -160,7 +174,18 @@ def _apply_attn_res(
block_write_idx,
norm.variance_epsilon,
0.0 if output_norm is None else output_norm.variance_epsilon,
quant_dtype=cast(torch.dtype, kFp8DynamicTokenSym.dtype)
if fuse_quant
else None,
)
if fuse_quant:
assert isinstance(result, tuple)
data, scale = result
return QuantizedActivation(
data, scale, prefix_sum.dtype, prefix_sum.shape, kFp8DynamicTokenSym
)
assert isinstance(result, torch.Tensor)
return result
class KimiMoE(nn.Module):
@@ -452,10 +477,13 @@ class KimiMLAAttention(nn.Module):
prefix,
)
def get_input_quant_key(self) -> QuantKey | None:
return self.mla_attn.get_input_quant_key()
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
hidden_states: torch.Tensor | QuantizedActivation,
) -> torch.Tensor:
return self.mla_attn(positions, hidden_states)
@@ -579,7 +607,7 @@ class KimiDecoderLayer(nn.Module):
def _run_self_attn(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
hidden_states: torch.Tensor | QuantizedActivation,
) -> torch.Tensor:
return self.self_attn(
hidden_states=hidden_states,
@@ -626,6 +654,11 @@ class KimiDecoderLayer(nn.Module):
prefix_delta: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
prefix_sum = hidden_states
attention_quant_key = (
self.self_attn.get_input_quant_key()
if isinstance(self.self_attn, (KimiK3DeltaAttention, KimiMLAAttention))
else None
)
hidden_states = _apply_attn_res(
prefix_sum,
block_residual,
@@ -635,6 +668,7 @@ class KimiDecoderLayer(nn.Module):
delta=prefix_delta,
output_norm=self.input_layernorm,
block_write_idx=(self.block_write_idx if self.is_block_write_layer else -1),
quant_key=attention_quant_key,
)
if self.is_block_write_layer:
@@ -659,6 +693,11 @@ class KimiDecoderLayer(nn.Module):
mlp_valid_blocks,
delta=prefix_delta,
output_norm=self.post_attention_layernorm,
quant_key=(
self.mlp.get_input_quant_key()
if isinstance(self.mlp, KimiMLP)
else None
),
)
hidden_states = self.mlp(hidden_states)
+27 -2
View File
@@ -8,8 +8,13 @@ import torch
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.layers.attention.attention import get_attention_context
from vllm.model_executor.layers.fusion.quant_activation import (
QuantizedActivation,
get_input_quant_key,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.mla import MultiHeadLatentAttentionWrapper
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
from vllm.platforms import current_platform
_OPT_KV_LORA_RANK = 512
@@ -49,6 +54,21 @@ class KimiK3MultiHeadLatentAttentionWrapper(MultiHeadLatentAttentionWrapper):
not attn.is_aiter_triton_fp8_bmm_enabled and attn.W_UK_T is not None
)
def get_input_quant_key(self) -> QuantKey | None:
if self.indexer is not None:
return None
consumers = (
[self.fused_qkv_a_proj]
if self.q_lora_rank is not None
else [self.kv_a_proj_with_mqa, self.q_proj]
)
if self.g_proj is not None:
consumers.append(self.g_proj)
if any(layer is None for layer in consumers):
return None
keys = [get_input_quant_key(layer) for layer in consumers if layer is not None]
return keys[0] if all(key == keys[0] for key in keys) else None
def _normalize_q_kv(
self,
q_c: torch.Tensor,
@@ -173,10 +193,15 @@ class KimiK3MultiHeadLatentAttentionWrapper(MultiHeadLatentAttentionWrapper):
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
hidden_states: torch.Tensor | QuantizedActivation,
llama_4_scaling: torch.Tensor | None = None,
) -> torch.Tensor:
q_c = None
num_tokens = (
hidden_states.orig_shape[0]
if isinstance(hidden_states, QuantizedActivation)
else hidden_states.shape[0]
)
if self.q_lora_rank is not None:
assert self.fused_qkv_a_proj is not None, (
@@ -238,7 +263,7 @@ class KimiK3MultiHeadLatentAttentionWrapper(MultiHeadLatentAttentionWrapper):
if self.dcp_q_replicate:
q_dcp_replicated, q = q, q_proj_layer._local_view(q)
output_shape = (hidden_states.shape[0], self.num_heads * self.v_head_dim)
output_shape = (num_tokens, self.num_heads * self.v_head_dim)
# The fused decode path covers only the MQA slice, so it is taken for
# decode-only batches; anything else falls through to MLAAttention.
+27 -4
View File
@@ -22,6 +22,7 @@ def _attn_res_kernel(
qk_weight_ptr,
output_norm_weight_ptr,
output_ptr,
output_scale_ptr,
stride_prefix_m: tl.constexpr,
stride_delta_m: tl.constexpr,
stride_block_m: tl.constexpr,
@@ -35,6 +36,7 @@ def _attn_res_kernel(
HAS_DELTA: tl.constexpr,
WRITE_BLOCK: tl.constexpr,
APPLY_OUTPUT_NORM: tl.constexpr,
QUANT_MAX: tl.constexpr,
BLOCK_L: tl.constexpr,
BLOCK_D: tl.constexpr,
):
@@ -129,6 +131,14 @@ def _attn_res_kernel(
output_norm_weight_ptr + d_offsets, mask=d_mask, other=0.0
).to(tl.float32)
output = mixed * output_reciprocal_std * output_norm_weight
if QUANT_MAX > 0:
# Preserve the rounding of the original BF16 output before quantizing.
output = output.to(prefix_ptr.dtype.element_ty).to(tl.float32)
amax = tl.max(tl.where(d_mask, tl.abs(output), 0.0), axis=0)
scale = tl.maximum(amax, 1e-10) * (1.0 / QUANT_MAX)
inv_scale = 1.0 / scale
output = tl.minimum(tl.maximum(output * inv_scale, -QUANT_MAX), QUANT_MAX)
tl.store(output_scale_ptr + row_idx, scale)
tl.store(
output_ptr + row_idx * stride_output_m + d_offsets,
output,
@@ -147,7 +157,9 @@ def attn_res(
block_write_idx: int,
eps: float,
output_norm_eps: float,
) -> torch.Tensor:
*,
quant_dtype: torch.dtype | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
num_tokens, hidden_size = prefix.shape
assert prefix.stride(-1) == 1
assert delta is None or delta.stride(-1) == 1
@@ -155,9 +167,18 @@ def attn_res(
assert norm_weight.stride(-1) == 1
assert qk_weight.stride(-1) == 1
assert output_norm_weight is None or output_norm_weight.stride(-1) == 1
output = prefix.new_empty(prefix.shape)
if quant_dtype is not None:
assert quant_dtype in (torch.float8_e4m3fn, torch.float8_e4m3fnuz)
output = torch.empty_like(
prefix, dtype=quant_dtype or prefix.dtype, memory_format=torch.contiguous_format
)
scale = (
torch.empty((num_tokens, 1), device=prefix.device, dtype=torch.float32)
if quant_dtype is not None
else None
)
if num_tokens == 0:
return output
return output if scale is None else (output, scale)
# Source tiling helps decode, while one-source tiles scale better for prefill.
if num_tokens >= 256 or num_blocks <= 1:
@@ -172,6 +193,7 @@ def attn_res(
qk_weight,
output_norm_weight,
output,
scale,
prefix.stride(0),
0 if delta is None else delta.stride(0),
blocks.stride(0),
@@ -185,9 +207,10 @@ def attn_res(
HAS_DELTA=delta is not None,
WRITE_BLOCK=block_write_idx >= 0,
APPLY_OUTPUT_NORM=output_norm_weight is not None,
QUANT_MAX=0.0 if quant_dtype is None else torch.finfo(quant_dtype).max,
BLOCK_L=block_l,
BLOCK_D=triton.next_power_of_2(hidden_size),
num_warps=num_warps,
num_stages=2,
)
return output
return output if scale is None else (output, scale)