[ROCm] Resolve the indexer fp8 cache dtype once at import (#53792)

Signed-off-by: Sriram Kumar <sriramkumar.kishorekumar@amd.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
Sriram Kumar
2026-09-17 17:51:35 +00:00
committed by GitHub
co-authored by Douglas Lehr
parent f3aa88d230
commit ff6b5808c4
+9 -11
View File
@@ -29,6 +29,8 @@ else:
logger = init_logger(__name__)
FP8_DTYPE = current_platform.fp8_dtype()
@functools.cache
def _get_aiter_topk_ops() -> tuple[Callable[..., None], Callable[..., None]] | None:
@@ -260,8 +262,7 @@ def indexer_k_quant_and_cache_triton(
# In real layout, we store the first portion as kv cache value
# and second portion as kv cache scale
kv_cache = kv_cache.view(num_blocks, -1)
fp8_dtype = current_platform.fp8_dtype()
kv_cache_value = kv_cache[:, : block_size * head_dim].view(fp8_dtype)
kv_cache_value = kv_cache[:, : block_size * head_dim].view(FP8_DTYPE)
kv_cache_scale = kv_cache[:, block_size * head_dim :].view(torch.float32)
head_tile_size = head_tile_size // kv_cache.element_size()
layout = "NORMAL" if block_size == 1 else "SHUFFLE"
@@ -279,7 +280,7 @@ def indexer_k_quant_and_cache_triton(
layout,
block_tile_size,
head_tile_size,
IS_FNUZ=current_platform.fp8_dtype() == torch.float8_e4m3fnuz,
IS_FNUZ=torch.float8_e4m3fnuz == FP8_DTYPE,
USE_UE8M0=scale_fmt == "ue8m0",
)
@@ -465,8 +466,7 @@ def cp_gather_indexer_k_quant_cache_triton(
num_blocks = k_cache.shape[0]
# we assume the kv cache already been split to 2 portion
k_cache = k_cache.view(num_blocks, -1)
fp8_dtype = current_platform.fp8_dtype()
k_cache_value = k_cache[:, : block_size * head_dim].view(fp8_dtype)
k_cache_value = k_cache[:, : block_size * head_dim].view(FP8_DTYPE)
k_cache_scale = k_cache[:, block_size * head_dim :].view(torch.float32)
grid = (num_tokens,)
k_fp8_scale = k_fp8_scale.view(torch.float32)
@@ -516,7 +516,6 @@ def fp8_paged_mqa_logits_torch(
):
from vllm.utils.math_utils import cdiv
fp8_dtype = current_platform.fp8_dtype()
batch_size, next_n, _, dim = q.size()
if next_n == 1:
block_size = kv_cache.shape[1]
@@ -540,7 +539,7 @@ def fp8_paged_mqa_logits_torch(
cache = kv_cache_flat[pages]
scale_offset = block_size * dim
cache_value = (
cache[..., :scale_offset].view(dtype=fp8_dtype).to(torch.float32)
cache[..., :scale_offset].view(dtype=FP8_DTYPE).to(torch.float32)
)
cache_scale = (
cache[..., scale_offset:].view(dtype=torch.float32).contiguous()
@@ -558,7 +557,7 @@ def fp8_paged_mqa_logits_torch(
kv_cache, scale = kv_cache[..., :dim], kv_cache[..., dim:]
scale = scale.contiguous().view(torch.float)
q = q.float()
kv_cache = kv_cache.view(fp8_dtype).float() * scale
kv_cache = kv_cache.view(FP8_DTYPE).float() * scale
num_block, block_size, _, dim = kv_cache.size()
logits = torch.full(
[batch_size * next_n, max_model_len],
@@ -1021,7 +1020,6 @@ def rocm_aiter_sparse_attn_indexer(
# careful! this will be None in dummy run
forward_context = get_forward_context()
attn_metadata = forward_context.attn_metadata
fp8_dtype = current_platform.fp8_dtype()
from vllm.utils.torch_utils import _resolve_layer_name
k_cache_prefix = _resolve_layer_name(k_cache_prefix)
@@ -1037,7 +1035,7 @@ def rocm_aiter_sparse_attn_indexer(
# Prefill k_fp8 and k_scale buffers, used by
# rocm_aiter_sparse_attn_indexer's prefill path
workspace_manager.get_simultaneous(
((total_seq_lens, head_dim), fp8_dtype),
((total_seq_lens, head_dim), FP8_DTYPE),
((total_seq_lens, 4), torch.uint8),
)
@@ -1117,7 +1115,7 @@ def rocm_aiter_sparse_attn_indexer(
workspace_manager = current_workspace_manager()
k_fp8_full, k_scale_full = workspace_manager.get_simultaneous(
((total_seq_lens, head_dim), fp8_dtype),
((total_seq_lens, head_dim), FP8_DTYPE),
((total_seq_lens, 4), torch.uint8),
)
for chunk in prefill_metadata.chunks: