[ROCm] Resolve the indexer fp8 cache dtype once at import (#53792)
Signed-off-by: Sriram Kumar <sriramkumar.kishorekumar@amd.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
co-authored by
Douglas Lehr
parent
f3aa88d230
commit
ff6b5808c4
@@ -29,6 +29,8 @@ else:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
|
||||
|
||||
@functools.cache
|
||||
def _get_aiter_topk_ops() -> tuple[Callable[..., None], Callable[..., None]] | None:
|
||||
@@ -260,8 +262,7 @@ def indexer_k_quant_and_cache_triton(
|
||||
# In real layout, we store the first portion as kv cache value
|
||||
# and second portion as kv cache scale
|
||||
kv_cache = kv_cache.view(num_blocks, -1)
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
kv_cache_value = kv_cache[:, : block_size * head_dim].view(fp8_dtype)
|
||||
kv_cache_value = kv_cache[:, : block_size * head_dim].view(FP8_DTYPE)
|
||||
kv_cache_scale = kv_cache[:, block_size * head_dim :].view(torch.float32)
|
||||
head_tile_size = head_tile_size // kv_cache.element_size()
|
||||
layout = "NORMAL" if block_size == 1 else "SHUFFLE"
|
||||
@@ -279,7 +280,7 @@ def indexer_k_quant_and_cache_triton(
|
||||
layout,
|
||||
block_tile_size,
|
||||
head_tile_size,
|
||||
IS_FNUZ=current_platform.fp8_dtype() == torch.float8_e4m3fnuz,
|
||||
IS_FNUZ=torch.float8_e4m3fnuz == FP8_DTYPE,
|
||||
USE_UE8M0=scale_fmt == "ue8m0",
|
||||
)
|
||||
|
||||
@@ -465,8 +466,7 @@ def cp_gather_indexer_k_quant_cache_triton(
|
||||
num_blocks = k_cache.shape[0]
|
||||
# we assume the kv cache already been split to 2 portion
|
||||
k_cache = k_cache.view(num_blocks, -1)
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
k_cache_value = k_cache[:, : block_size * head_dim].view(fp8_dtype)
|
||||
k_cache_value = k_cache[:, : block_size * head_dim].view(FP8_DTYPE)
|
||||
k_cache_scale = k_cache[:, block_size * head_dim :].view(torch.float32)
|
||||
grid = (num_tokens,)
|
||||
k_fp8_scale = k_fp8_scale.view(torch.float32)
|
||||
@@ -516,7 +516,6 @@ def fp8_paged_mqa_logits_torch(
|
||||
):
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
batch_size, next_n, _, dim = q.size()
|
||||
if next_n == 1:
|
||||
block_size = kv_cache.shape[1]
|
||||
@@ -540,7 +539,7 @@ def fp8_paged_mqa_logits_torch(
|
||||
cache = kv_cache_flat[pages]
|
||||
scale_offset = block_size * dim
|
||||
cache_value = (
|
||||
cache[..., :scale_offset].view(dtype=fp8_dtype).to(torch.float32)
|
||||
cache[..., :scale_offset].view(dtype=FP8_DTYPE).to(torch.float32)
|
||||
)
|
||||
cache_scale = (
|
||||
cache[..., scale_offset:].view(dtype=torch.float32).contiguous()
|
||||
@@ -558,7 +557,7 @@ def fp8_paged_mqa_logits_torch(
|
||||
kv_cache, scale = kv_cache[..., :dim], kv_cache[..., dim:]
|
||||
scale = scale.contiguous().view(torch.float)
|
||||
q = q.float()
|
||||
kv_cache = kv_cache.view(fp8_dtype).float() * scale
|
||||
kv_cache = kv_cache.view(FP8_DTYPE).float() * scale
|
||||
num_block, block_size, _, dim = kv_cache.size()
|
||||
logits = torch.full(
|
||||
[batch_size * next_n, max_model_len],
|
||||
@@ -1021,7 +1020,6 @@ def rocm_aiter_sparse_attn_indexer(
|
||||
# careful! this will be None in dummy run
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
from vllm.utils.torch_utils import _resolve_layer_name
|
||||
|
||||
k_cache_prefix = _resolve_layer_name(k_cache_prefix)
|
||||
@@ -1037,7 +1035,7 @@ def rocm_aiter_sparse_attn_indexer(
|
||||
# Prefill k_fp8 and k_scale buffers, used by
|
||||
# rocm_aiter_sparse_attn_indexer's prefill path
|
||||
workspace_manager.get_simultaneous(
|
||||
((total_seq_lens, head_dim), fp8_dtype),
|
||||
((total_seq_lens, head_dim), FP8_DTYPE),
|
||||
((total_seq_lens, 4), torch.uint8),
|
||||
)
|
||||
|
||||
@@ -1117,7 +1115,7 @@ def rocm_aiter_sparse_attn_indexer(
|
||||
|
||||
workspace_manager = current_workspace_manager()
|
||||
k_fp8_full, k_scale_full = workspace_manager.get_simultaneous(
|
||||
((total_seq_lens, head_dim), fp8_dtype),
|
||||
((total_seq_lens, head_dim), FP8_DTYPE),
|
||||
((total_seq_lens, 4), torch.uint8),
|
||||
)
|
||||
for chunk in prefill_metadata.chunks:
|
||||
|
||||
Reference in New Issue
Block a user