diff --git a/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py b/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py index 07cdf18106..3d891c8ce0 100644 --- a/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py +++ b/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py @@ -29,6 +29,8 @@ else: logger = init_logger(__name__) +FP8_DTYPE = current_platform.fp8_dtype() + @functools.cache def _get_aiter_topk_ops() -> tuple[Callable[..., None], Callable[..., None]] | None: @@ -260,8 +262,7 @@ def indexer_k_quant_and_cache_triton( # In real layout, we store the first portion as kv cache value # and second portion as kv cache scale kv_cache = kv_cache.view(num_blocks, -1) - fp8_dtype = current_platform.fp8_dtype() - kv_cache_value = kv_cache[:, : block_size * head_dim].view(fp8_dtype) + kv_cache_value = kv_cache[:, : block_size * head_dim].view(FP8_DTYPE) kv_cache_scale = kv_cache[:, block_size * head_dim :].view(torch.float32) head_tile_size = head_tile_size // kv_cache.element_size() layout = "NORMAL" if block_size == 1 else "SHUFFLE" @@ -279,7 +280,7 @@ def indexer_k_quant_and_cache_triton( layout, block_tile_size, head_tile_size, - IS_FNUZ=current_platform.fp8_dtype() == torch.float8_e4m3fnuz, + IS_FNUZ=torch.float8_e4m3fnuz == FP8_DTYPE, USE_UE8M0=scale_fmt == "ue8m0", ) @@ -465,8 +466,7 @@ def cp_gather_indexer_k_quant_cache_triton( num_blocks = k_cache.shape[0] # we assume the kv cache already been split to 2 portion k_cache = k_cache.view(num_blocks, -1) - fp8_dtype = current_platform.fp8_dtype() - k_cache_value = k_cache[:, : block_size * head_dim].view(fp8_dtype) + k_cache_value = k_cache[:, : block_size * head_dim].view(FP8_DTYPE) k_cache_scale = k_cache[:, block_size * head_dim :].view(torch.float32) grid = (num_tokens,) k_fp8_scale = k_fp8_scale.view(torch.float32) @@ -516,7 +516,6 @@ def fp8_paged_mqa_logits_torch( ): from vllm.utils.math_utils import cdiv - fp8_dtype = current_platform.fp8_dtype() batch_size, next_n, _, dim = q.size() if next_n == 1: block_size = kv_cache.shape[1] @@ -540,7 +539,7 @@ def fp8_paged_mqa_logits_torch( cache = kv_cache_flat[pages] scale_offset = block_size * dim cache_value = ( - cache[..., :scale_offset].view(dtype=fp8_dtype).to(torch.float32) + cache[..., :scale_offset].view(dtype=FP8_DTYPE).to(torch.float32) ) cache_scale = ( cache[..., scale_offset:].view(dtype=torch.float32).contiguous() @@ -558,7 +557,7 @@ def fp8_paged_mqa_logits_torch( kv_cache, scale = kv_cache[..., :dim], kv_cache[..., dim:] scale = scale.contiguous().view(torch.float) q = q.float() - kv_cache = kv_cache.view(fp8_dtype).float() * scale + kv_cache = kv_cache.view(FP8_DTYPE).float() * scale num_block, block_size, _, dim = kv_cache.size() logits = torch.full( [batch_size * next_n, max_model_len], @@ -1021,7 +1020,6 @@ def rocm_aiter_sparse_attn_indexer( # careful! this will be None in dummy run forward_context = get_forward_context() attn_metadata = forward_context.attn_metadata - fp8_dtype = current_platform.fp8_dtype() from vllm.utils.torch_utils import _resolve_layer_name k_cache_prefix = _resolve_layer_name(k_cache_prefix) @@ -1037,7 +1035,7 @@ def rocm_aiter_sparse_attn_indexer( # Prefill k_fp8 and k_scale buffers, used by # rocm_aiter_sparse_attn_indexer's prefill path workspace_manager.get_simultaneous( - ((total_seq_lens, head_dim), fp8_dtype), + ((total_seq_lens, head_dim), FP8_DTYPE), ((total_seq_lens, 4), torch.uint8), ) @@ -1117,7 +1115,7 @@ def rocm_aiter_sparse_attn_indexer( workspace_manager = current_workspace_manager() k_fp8_full, k_scale_full = workspace_manager.get_simultaneous( - ((total_seq_lens, head_dim), fp8_dtype), + ((total_seq_lens, head_dim), FP8_DTYPE), ((total_seq_lens, 4), torch.uint8), ) for chunk in prefill_metadata.chunks: