diff --git a/benchmarks/kernels/benchmark_kimi_k3_amd_fp8_fusions.py b/benchmarks/kernels/benchmark_kimi_k3_amd_fp8_fusions.py new file mode 100644 index 0000000000..dd210ae55d --- /dev/null +++ b/benchmarks/kernels/benchmark_kimi_k3_amd_fp8_fusions.py @@ -0,0 +1,165 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Eight-GPU AttnRes+FP8 projection comparison with rotated graph timing.""" + +import argparse +import json +import os +import statistics +from pathlib import Path + +import torch +import torch.distributed as dist + +from vllm._aiter_ops import rocm_aiter_ops +from vllm.config import VllmConfig, set_current_vllm_config +from vllm.models.kimi_k3.amd.ops.attn_res import attn_res +from vllm.platforms import current_platform + + +def capture(fn, repeats): + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + for _ in range(5): + fn() + torch.cuda.current_stream().wait_stream(stream) + torch.accelerator.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + for _ in range(repeats): + fn() + return graph + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument( + "--tokens", type=int, nargs="+", default=[1, 4, 16, 64, 256, 1024, 4096, 16384] + ) + parser.add_argument("--blocks", type=int, nargs="+", default=[1, 4, 8]) + args = parser.parse_args() + assert int(os.environ["WORLD_SIZE"]) == 8 + assert torch.accelerator.device_count() == 8 + rank = int(os.environ["LOCAL_RANK"]) + torch.accelerator.set_device_index(rank) + torch.set_num_threads(1) + torch.manual_seed(1729 + rank) + dist.init_process_group("gloo") + args.output.mkdir(parents=True, exist_ok=True) + dtype = current_platform.fp8_dtype() + rows = [] + with set_current_vllm_config(VllmConfig()): + weight = torch.randn(2112, 7168, device="cuda", dtype=torch.bfloat16) * 0.02 + weight, ws = rocm_aiter_ops.per_token_quant(weight, dtype) + weight = rocm_aiter_ops.shuffle_weight(weight) + for m in args.tokens: + prefix = torch.randn(m, 7168, device="cuda", dtype=torch.bfloat16) + delta = torch.zeros_like(prefix) + blocks = torch.randn(m, 9, 7168, device="cuda", dtype=torch.bfloat16) + norm = torch.ones(7168, device="cuda", dtype=torch.bfloat16) + score = torch.randn_like(norm) / 7168**0.5 + for b in args.blocks: + + def produce( + fused, + prefix=prefix, + delta=delta, + blocks=blocks, + norm=norm, + score=score, + b=b, + ): + result = attn_res( + prefix, + delta, + blocks, + norm, + score, + norm, + b, + -1, + 1e-5, + 1e-5, + quant_dtype=dtype if fused else None, + ) + return ( + result + if fused + else rocm_aiter_ops.per_token_quant(result, dtype) + ) + + def segment(fused, produce=produce): + x, scale = produce(fused) + return rocm_aiter_ops.preshuffled_per_token_w8a8_gemm( + x, weight, scale, ws, output_dtype=torch.bfloat16 + ) + + expected, actual = segment(False), segment(True) + relative_l2 = ( + (actual.float() - expected.float()).norm() / expected.float().norm() + ).item() + assert relative_l2 < 0.002, relative_l2 + repeats = 20 if m < 1024 else 4 + functions = { + "baseline_producer": lambda produce=produce: produce(False), + "fused_producer": lambda produce=produce: produce(True), + "baseline_segment": lambda segment=segment: segment(False), + "fused_segment": lambda segment=segment: segment(True), + } + graphs = {k: capture(fn, repeats) for k, fn in functions.items()} + samples = {k: [] for k in functions} + names = list(functions) + for sample in range(9): + for name in names[sample % 4 :] + names[: sample % 4]: + dist.barrier() + graphs[name].replay() + torch.accelerator.synchronize() + start, end = [ + torch.cuda.Event(enable_timing=True) for _ in range(2) + ] + start.record() + graphs[name].replay() + end.record() + end.synchronize() + samples[name].append(start.elapsed_time(end) * 1000 / repeats) + row = { + "tokens": m, + "blocks": b, + "relative_l2": relative_l2, + "samples_us": samples, + "median_us": {k: statistics.median(v) for k, v in samples.items()}, + } + rows.append(row) + print( + json.dumps( + { + "rank": rank, + "tokens": m, + "blocks": b, + "median_us": row["median_us"], + } + ), + flush=True, + ) + del graphs + (args.output / f"rank-{rank}.json").write_text( + json.dumps( + { + "rank": rank, + "world_size": 8, + "torch": torch.__version__, + "rows": rows, + "scope": "AttnRes + FP8 quant + TP8 MLA input projection", + }, + indent=2, + ) + + "\n" + ) + dist.barrier() + dist.destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/tests/models/kimi_k3/test_amd_attn_res.py b/tests/models/kimi_k3/test_amd_attn_res.py index 80d5e1936f..f4efdbd5b2 100644 --- a/tests/models/kimi_k3/test_amd_attn_res.py +++ b/tests/models/kimi_k3/test_amd_attn_res.py @@ -191,3 +191,102 @@ def test_amd_attn_res_fused_contract( original_blocks[:, block_write_idx].copy_(expected_prefix) torch.testing.assert_close(blocks, original_blocks, atol=0, rtol=0) assert actual.is_contiguous() + + +@pytest.mark.parametrize( + "num_tokens,num_blocks,has_delta,write_block", + [ + (0, 0, False, False), + (1, 0, True, True), + (17, 4, True, True), + (320, 8, False, False), + ], +) +def test_amd_attn_res_fp8_preserves_prefix_and_quantized_output( + num_tokens, + num_blocks, + has_delta, + write_block, + default_vllm_config, +): + from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8 + from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape + + torch.manual_seed(41) + hidden_size = 7168 + prefix = _randn_with_row_padding(num_tokens, hidden_size, padding=7) + delta = _randn_with_row_padding(num_tokens, hidden_size) if has_delta else None + blocks = _randn_with_row_padding(num_tokens, 9, hidden_size, padding=13) + norm = 1 + torch.randn(hidden_size, device="cuda", dtype=torch.bfloat16) * 0.1 + score = torch.randn_like(norm) / hidden_size**0.5 + out_norm = 1 + torch.randn_like(norm) * 0.1 + ref_prefix, ref_blocks = prefix.clone(), blocks.clone() + kwargs = dict( + num_blocks=num_blocks, + block_write_idx=num_blocks if write_block else -1, + eps=1e-5, + output_norm_eps=1e-5, + ) + reference = attn_res(ref_prefix, delta, ref_blocks, norm, score, out_norm, **kwargs) + output, scale = attn_res( + prefix, + delta, + blocks, + norm, + score, + out_norm, + quant_dtype=current_platform.fp8_dtype(), + **kwargs, + ) + assert output.dtype == current_platform.fp8_dtype() + assert scale.shape == (num_tokens, 1) + if num_tokens: + # Keep reference rounding independent of Inductor's fused quantization. + quant = QuantFP8( + static=False, group_shape=GroupShape.PER_TOKEN, compile_native=False + ) + ref_output, ref_scale = quant(reference) + torch.testing.assert_close(scale, ref_scale, atol=1e-7, rtol=1e-6) + # Floating-point fusion can move values at FP8 rounding midpoints. + # Bound each change to the adjacent code and bound how often it occurs. + code_delta = ( + output.view(torch.uint8).to(torch.int16) + - ref_output.view(torch.uint8).to(torch.int16) + ).abs() + assert code_delta.max().item() <= 1 + assert (code_delta != 0).float().mean().item() < 1e-4 + torch.testing.assert_close(prefix, ref_prefix, atol=0, rtol=0) + torch.testing.assert_close(blocks, ref_blocks, atol=0, rtol=0) + + +@pytest.mark.parametrize( + "matching,requires_unquantized", [(True, False), (False, False), (True, True)] +) +def test_mla_input_quant_requires_all_consumers(matching, requires_unquantized): + from types import SimpleNamespace + + from torch import nn + + from vllm.model_executor.layers.fusion.quant_activation import ( + expose_input_quant_key, + ) + from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kFp8DynamicTokenSym, + ) + from vllm.models.kimi_k3.amd.mla import KimiK3MultiHeadLatentAttentionWrapper + + attention = object.__new__(KimiK3MultiHeadLatentAttentionWrapper) + nn.Module.__init__(attention) + attention.indexer = None + attention.q_lora_rank = 1536 + attention.fused_qkv_a_proj = nn.Identity() + attention.g_proj = nn.Identity() + kernel = SimpleNamespace(input_quant_key=lambda: kFp8DynamicTokenSym) + expose_input_quant_key(attention.fused_qkv_a_proj, kernel) + if matching: + expose_input_quant_key(attention.g_proj, kernel) + attention.g_proj.requires_unquantized_input = requires_unquantized + actual = attention.get_input_quant_key() + assert actual == ( + kFp8DynamicTokenSym if matching and not requires_unquantized else None + ) diff --git a/tests/models/kimi_k3/test_amd_mla_direct_return.py b/tests/models/kimi_k3/test_amd_mla_direct_return.py index a1c120737d..50a52f4fe1 100644 --- a/tests/models/kimi_k3/test_amd_mla_direct_return.py +++ b/tests/models/kimi_k3/test_amd_mla_direct_return.py @@ -1,6 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from types import SimpleNamespace + +import pytest import torch from torch import nn @@ -36,3 +39,81 @@ def test_mla_self_attention_returns_projection_storage_directly() -> None: assert output is projected assert output.data_ptr() == projected.data_ptr() + + +@pytest.mark.parametrize("quantized", [False, True]) +@pytest.mark.parametrize("fused_decode", [False, True]) +def test_mla_forward_preserves_output_shape_with_fp8_input( + monkeypatch, quantized: bool, fused_decode: bool +) -> None: + """Both MLA paths accept producer-quantized input after decode fusion.""" + from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation + from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kFp8DynamicTokenSym, + ) + from vllm.models.kimi_k3.amd.mla import KimiK3MultiHeadLatentAttentionWrapper + + hidden_states = torch.randn(4, 8, dtype=torch.bfloat16) + if quantized: + hidden_states = QuantizedActivation( + hidden_states.to(kFp8DynamicTokenSym.dtype), + torch.ones(4, 1), + hidden_states.dtype, + hidden_states.shape, + kFp8DynamicTokenSym, + ) + projected = torch.randn(4, 8, dtype=torch.bfloat16) + paths = [] + + def input_projection(value): + assert value is hidden_states + return torch.zeros(4, 8, dtype=torch.bfloat16), None + + def attention_output(path, output_shape): + assert output_shape == projected.shape + paths.append(path) + return projected + + attention = SimpleNamespace( + q_lora_rank=2, + kv_lora_rank=4, + qk_rope_head_dim=2, + qk_head_dim=4, + num_heads=2, + v_head_dim=4, + fused_qkv_a_proj=input_projection, + q_a_layernorm=nn.Identity(), + _normalize_q_kv=lambda q, kv: (q, kv), + q_b_proj=lambda q: (torch.zeros(4, 8, dtype=q.dtype), None), + rotary_emb=None, + indexer=None, + dcp_q_replicate=False, + _fused_qk_prep=fused_decode, + _fused_decode=lambda q, kv, pe, pos, shape, *args: attention_output( + "fused", shape + ), + g_proj=None, + o_proj=lambda value: (value, None), + ) + if fused_decode: + attention.mla_attn = SimpleNamespace(layer_name="test") + monkeypatch.setattr( + "vllm.models.kimi_k3.amd.mla.get_attention_context", + lambda _: ( + SimpleNamespace(num_actual_tokens=4, num_decode_tokens=4), + None, + torch.zeros(1), + torch.arange(4), + ), + ) + else: + attention.mla_attn = lambda q, kv, pe, **kwargs: attention_output( + "regular", kwargs["output_shape"] + ) + + output = KimiK3MultiHeadLatentAttentionWrapper.forward( + attention, torch.arange(4), hidden_states + ) + + assert output is projected + assert paths == ["fused" if fused_decode else "regular"] diff --git a/vllm/models/kimi_k3/amd/kda.py b/vllm/models/kimi_k3/amd/kda.py index e2dea74b6b..409cf6b1bf 100644 --- a/vllm/models/kimi_k3/amd/kda.py +++ b/vllm/models/kimi_k3/amd/kda.py @@ -13,6 +13,10 @@ from vllm.config import VllmConfig from vllm.distributed import divide from vllm.forward_context import get_forward_context from vllm.logger import init_logger +from vllm.model_executor.layers.fusion.quant_activation import ( + QuantizedActivation, + get_input_quant_key, +) from vllm.model_executor.layers.linear import ( ColumnParallelLinear, RowParallelLinear, @@ -37,6 +41,7 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import ( causal_conv1d_fn, causal_conv1d_update, ) +from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader from vllm.model_executor.utils import set_weight_attrs from vllm.models.kimi_k3.amd.kda_metadata import KimiK3ROCmKDABackend @@ -300,13 +305,16 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention): qkv = qkv.permute(1, 0, 2, 3).contiguous().unsqueeze(1) return qkv.unbind(0) + def get_input_quant_key(self) -> QuantKey | None: + return get_input_quant_key(self.in_proj_qkvgfab) + def forward( self, - hidden_states: torch.Tensor, + hidden_states: torch.Tensor | QuantizedActivation, positions: torch.Tensor, ) -> torch.Tensor: - num_tokens = hidden_states.size(0) projected_qkvgfab = self.in_proj_qkvgfab(hidden_states)[0] + num_tokens = projected_qkvgfab.shape[0] split_sizes = [ 3 * self.local_projection_size, @@ -326,8 +334,8 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention): core_attn_out = torch.empty( (1, num_tokens, self.local_num_heads, self.head_dim), - dtype=hidden_states.dtype, - device=hidden_states.device, + dtype=projected_qkvgfab.dtype, + device=projected_qkvgfab.device, ) self._forward( diff --git a/vllm/models/kimi_k3/amd/linear.py b/vllm/models/kimi_k3/amd/linear.py index 65dad661ab..3a1d827abe 100644 --- a/vllm/models/kimi_k3/amd/linear.py +++ b/vllm/models/kimi_k3/amd/linear.py @@ -2,11 +2,12 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from collections.abc import Iterable -from typing import Any +from typing import Any, cast import torch from torch import nn +from vllm import envs from vllm.config import CacheConfig, VllmConfig from vllm.distributed import ( get_pp_group, @@ -19,6 +20,10 @@ from vllm.model_executor.layers.fused_moe import ( fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear +from vllm.model_executor.layers.fusion.quant_activation import ( + QuantizedActivation, + get_input_quant_key, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -38,6 +43,10 @@ from vllm.model_executor.layers.mamba.mamba_utils import ( ) from vllm.model_executor.layers.mla import MLAModules from vllm.model_executor.layers.quantization.base_config import QuantizationConfig +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + QuantKey, + kFp8DynamicTokenSym, +) from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, @@ -114,6 +123,9 @@ class KimiMLP(nn.Module): "Only silu and situ are supported." ) + def get_input_quant_key(self) -> QuantKey | None: + return get_input_quant_key(self.gate_up_proj) + def forward(self, x): gate_up, _ = self.gate_up_proj(x) x = self.act_fn(gate_up) @@ -148,8 +160,10 @@ def _apply_attn_res( delta: torch.Tensor | None = None, output_norm: RMSNorm | None = None, block_write_idx: int = -1, -) -> torch.Tensor: - return attn_res( + quant_key: QuantKey | None = None, +) -> torch.Tensor | QuantizedActivation: + fuse_quant = quant_key == kFp8DynamicTokenSym and not envs.VLLM_BATCH_INVARIANT + result = attn_res( prefix_sum, delta, block_residual, @@ -160,7 +174,18 @@ def _apply_attn_res( block_write_idx, norm.variance_epsilon, 0.0 if output_norm is None else output_norm.variance_epsilon, + quant_dtype=cast(torch.dtype, kFp8DynamicTokenSym.dtype) + if fuse_quant + else None, ) + if fuse_quant: + assert isinstance(result, tuple) + data, scale = result + return QuantizedActivation( + data, scale, prefix_sum.dtype, prefix_sum.shape, kFp8DynamicTokenSym + ) + assert isinstance(result, torch.Tensor) + return result class KimiMoE(nn.Module): @@ -452,10 +477,13 @@ class KimiMLAAttention(nn.Module): prefix, ) + def get_input_quant_key(self) -> QuantKey | None: + return self.mla_attn.get_input_quant_key() + def forward( self, positions: torch.Tensor, - hidden_states: torch.Tensor, + hidden_states: torch.Tensor | QuantizedActivation, ) -> torch.Tensor: return self.mla_attn(positions, hidden_states) @@ -579,7 +607,7 @@ class KimiDecoderLayer(nn.Module): def _run_self_attn( self, positions: torch.Tensor, - hidden_states: torch.Tensor, + hidden_states: torch.Tensor | QuantizedActivation, ) -> torch.Tensor: return self.self_attn( hidden_states=hidden_states, @@ -626,6 +654,11 @@ class KimiDecoderLayer(nn.Module): prefix_delta: torch.Tensor | None, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: prefix_sum = hidden_states + attention_quant_key = ( + self.self_attn.get_input_quant_key() + if isinstance(self.self_attn, (KimiK3DeltaAttention, KimiMLAAttention)) + else None + ) hidden_states = _apply_attn_res( prefix_sum, block_residual, @@ -635,6 +668,7 @@ class KimiDecoderLayer(nn.Module): delta=prefix_delta, output_norm=self.input_layernorm, block_write_idx=(self.block_write_idx if self.is_block_write_layer else -1), + quant_key=attention_quant_key, ) if self.is_block_write_layer: @@ -659,6 +693,11 @@ class KimiDecoderLayer(nn.Module): mlp_valid_blocks, delta=prefix_delta, output_norm=self.post_attention_layernorm, + quant_key=( + self.mlp.get_input_quant_key() + if isinstance(self.mlp, KimiMLP) + else None + ), ) hidden_states = self.mlp(hidden_states) diff --git a/vllm/models/kimi_k3/amd/mla.py b/vllm/models/kimi_k3/amd/mla.py index 11df7d035f..1e4a51e8bf 100644 --- a/vllm/models/kimi_k3/amd/mla.py +++ b/vllm/models/kimi_k3/amd/mla.py @@ -8,8 +8,13 @@ import torch from vllm._aiter_ops import rocm_aiter_ops from vllm.model_executor.layers.attention.attention import get_attention_context +from vllm.model_executor.layers.fusion.quant_activation import ( + QuantizedActivation, + get_input_quant_key, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.mla import MultiHeadLatentAttentionWrapper +from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey from vllm.platforms import current_platform _OPT_KV_LORA_RANK = 512 @@ -49,6 +54,21 @@ class KimiK3MultiHeadLatentAttentionWrapper(MultiHeadLatentAttentionWrapper): not attn.is_aiter_triton_fp8_bmm_enabled and attn.W_UK_T is not None ) + def get_input_quant_key(self) -> QuantKey | None: + if self.indexer is not None: + return None + consumers = ( + [self.fused_qkv_a_proj] + if self.q_lora_rank is not None + else [self.kv_a_proj_with_mqa, self.q_proj] + ) + if self.g_proj is not None: + consumers.append(self.g_proj) + if any(layer is None for layer in consumers): + return None + keys = [get_input_quant_key(layer) for layer in consumers if layer is not None] + return keys[0] if all(key == keys[0] for key in keys) else None + def _normalize_q_kv( self, q_c: torch.Tensor, @@ -173,10 +193,15 @@ class KimiK3MultiHeadLatentAttentionWrapper(MultiHeadLatentAttentionWrapper): def forward( self, positions: torch.Tensor, - hidden_states: torch.Tensor, + hidden_states: torch.Tensor | QuantizedActivation, llama_4_scaling: torch.Tensor | None = None, ) -> torch.Tensor: q_c = None + num_tokens = ( + hidden_states.orig_shape[0] + if isinstance(hidden_states, QuantizedActivation) + else hidden_states.shape[0] + ) if self.q_lora_rank is not None: assert self.fused_qkv_a_proj is not None, ( @@ -238,7 +263,7 @@ class KimiK3MultiHeadLatentAttentionWrapper(MultiHeadLatentAttentionWrapper): if self.dcp_q_replicate: q_dcp_replicated, q = q, q_proj_layer._local_view(q) - output_shape = (hidden_states.shape[0], self.num_heads * self.v_head_dim) + output_shape = (num_tokens, self.num_heads * self.v_head_dim) # The fused decode path covers only the MQA slice, so it is taken for # decode-only batches; anything else falls through to MLAAttention. diff --git a/vllm/models/kimi_k3/amd/ops/attn_res.py b/vllm/models/kimi_k3/amd/ops/attn_res.py index 5802e60367..5c4fe7a0b0 100644 --- a/vllm/models/kimi_k3/amd/ops/attn_res.py +++ b/vllm/models/kimi_k3/amd/ops/attn_res.py @@ -22,6 +22,7 @@ def _attn_res_kernel( qk_weight_ptr, output_norm_weight_ptr, output_ptr, + output_scale_ptr, stride_prefix_m: tl.constexpr, stride_delta_m: tl.constexpr, stride_block_m: tl.constexpr, @@ -35,6 +36,7 @@ def _attn_res_kernel( HAS_DELTA: tl.constexpr, WRITE_BLOCK: tl.constexpr, APPLY_OUTPUT_NORM: tl.constexpr, + QUANT_MAX: tl.constexpr, BLOCK_L: tl.constexpr, BLOCK_D: tl.constexpr, ): @@ -129,6 +131,14 @@ def _attn_res_kernel( output_norm_weight_ptr + d_offsets, mask=d_mask, other=0.0 ).to(tl.float32) output = mixed * output_reciprocal_std * output_norm_weight + if QUANT_MAX > 0: + # Preserve the rounding of the original BF16 output before quantizing. + output = output.to(prefix_ptr.dtype.element_ty).to(tl.float32) + amax = tl.max(tl.where(d_mask, tl.abs(output), 0.0), axis=0) + scale = tl.maximum(amax, 1e-10) * (1.0 / QUANT_MAX) + inv_scale = 1.0 / scale + output = tl.minimum(tl.maximum(output * inv_scale, -QUANT_MAX), QUANT_MAX) + tl.store(output_scale_ptr + row_idx, scale) tl.store( output_ptr + row_idx * stride_output_m + d_offsets, output, @@ -147,7 +157,9 @@ def attn_res( block_write_idx: int, eps: float, output_norm_eps: float, -) -> torch.Tensor: + *, + quant_dtype: torch.dtype | None = None, +) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: num_tokens, hidden_size = prefix.shape assert prefix.stride(-1) == 1 assert delta is None or delta.stride(-1) == 1 @@ -155,9 +167,18 @@ def attn_res( assert norm_weight.stride(-1) == 1 assert qk_weight.stride(-1) == 1 assert output_norm_weight is None or output_norm_weight.stride(-1) == 1 - output = prefix.new_empty(prefix.shape) + if quant_dtype is not None: + assert quant_dtype in (torch.float8_e4m3fn, torch.float8_e4m3fnuz) + output = torch.empty_like( + prefix, dtype=quant_dtype or prefix.dtype, memory_format=torch.contiguous_format + ) + scale = ( + torch.empty((num_tokens, 1), device=prefix.device, dtype=torch.float32) + if quant_dtype is not None + else None + ) if num_tokens == 0: - return output + return output if scale is None else (output, scale) # Source tiling helps decode, while one-source tiles scale better for prefill. if num_tokens >= 256 or num_blocks <= 1: @@ -172,6 +193,7 @@ def attn_res( qk_weight, output_norm_weight, output, + scale, prefix.stride(0), 0 if delta is None else delta.stride(0), blocks.stride(0), @@ -185,9 +207,10 @@ def attn_res( HAS_DELTA=delta is not None, WRITE_BLOCK=block_write_idx >= 0, APPLY_OUTPUT_NORM=output_norm_weight is not None, + QUANT_MAX=0.0 if quant_dtype is None else torch.finfo(quant_dtype).max, BLOCK_L=block_l, BLOCK_D=triton.next_power_of_2(hidden_size), num_warps=num_warps, num_stages=2, ) - return output + return output if scale is None else (output, scale)