Llamas 3.1 405B fp4 changes upstreaming from 355_wip (#25135)
Signed-off-by: Aleksandr Malyshev <maleksan@amd.com> Co-authored-by: Aleksandr Malyshev <maleksan@amd.com> Co-authored-by: Doug Lehr <douglehr@amd.com>
This commit is contained in:
co-authored by
Aleksandr Malyshev
Doug Lehr
parent
8b77328ffe
commit
53a30845be
@@ -8,6 +8,8 @@ import torch
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
|
||||
from .common import apply_rotary_emb_torch
|
||||
from .rocm_aiter_rope_ops import (is_rocm_triton_rotary_embedding_enabled,
|
||||
rocm_aiter_rotary_emb)
|
||||
|
||||
|
||||
@CustomOp.register("rotary_embedding")
|
||||
@@ -45,6 +47,8 @@ class RotaryEmbedding(CustomOp):
|
||||
cache = cache.to(dtype)
|
||||
self.cos_sin_cache: torch.Tensor
|
||||
self.register_buffer("cos_sin_cache", cache, persistent=False)
|
||||
self.is_rocm_triton_rotary_embedding_enabled = \
|
||||
is_rocm_triton_rotary_embedding_enabled()
|
||||
|
||||
def _compute_inv_freq(self, base: float) -> torch.Tensor:
|
||||
"""Compute the inverse frequency."""
|
||||
@@ -120,14 +124,31 @@ class RotaryEmbedding(CustomOp):
|
||||
return query, key
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
|
||||
self._match_cos_sin_cache_dtype(query)
|
||||
|
||||
# ops.rotary_embedding() is an in-place operation
|
||||
# that updates the query and key tensors.
|
||||
ops.rotary_embedding(positions, query, key, self.head_size,
|
||||
self.cos_sin_cache, self.is_neox_style)
|
||||
return query, key
|
||||
|
||||
def forward_hip(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: Optional[torch.Tensor] = None,
|
||||
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
if self.is_rocm_triton_rotary_embedding_enabled:
|
||||
self._match_cos_sin_cache_dtype(query)
|
||||
rocm_aiter_rotary_emb(positions, query, key, self.cos_sin_cache,
|
||||
self.head_size, self.rotary_dim,
|
||||
self.is_neox_style)
|
||||
else:
|
||||
# ops.rotary_embedding() is an in-place operation
|
||||
# that updates the query and key tensors.
|
||||
self.forward_cuda(positions, query, key)
|
||||
return query, key
|
||||
|
||||
def forward_xpu(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils import direct_register_custom_op
|
||||
|
||||
|
||||
def is_rocm_triton_rotary_embedding_enabled() -> bool:
|
||||
return (current_platform.is_rocm() and envs.VLLM_ROCM_USE_AITER
|
||||
and envs.VLLM_ROCM_USE_TRITON_ROPE)
|
||||
|
||||
|
||||
def rocm_aiter_rotary_emb_with_key_forward_triton_impl(
|
||||
positions: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
cos: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
rotate_style: int = 0,
|
||||
is_nope_first: bool = False,
|
||||
) -> None:
|
||||
import aiter.ops.triton.rope as ops
|
||||
ops.rope_cached_thd_positions_2c_fwd_inplace(
|
||||
query,
|
||||
key,
|
||||
cos,
|
||||
sin,
|
||||
positions,
|
||||
rotate_style,
|
||||
reuse_freqs_front_part=True,
|
||||
nope_first=is_nope_first,
|
||||
)
|
||||
|
||||
|
||||
def rocm_aiter_rotary_emb_with_key_forward_triton_fake(
|
||||
positions: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
cos: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
rotate_style: int = 0,
|
||||
is_nope_first: bool = False,
|
||||
) -> None:
|
||||
pass
|
||||
|
||||
|
||||
if is_rocm_triton_rotary_embedding_enabled():
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="rocm_aiter_rotary_emb_with_key_forward_triton",
|
||||
op_func=rocm_aiter_rotary_emb_with_key_forward_triton_impl,
|
||||
mutates_args=["key", "query"],
|
||||
fake_impl=rocm_aiter_rotary_emb_with_key_forward_triton_fake,
|
||||
dispatch_key=current_platform.dispatch_key,
|
||||
)
|
||||
|
||||
|
||||
def rocm_aiter_rotary_emb(positions: torch.Tensor, query: torch.Tensor,
|
||||
key: torch.Tensor, cos_sin_cache: torch.Tensor,
|
||||
head_size: int, rotary_dim: int,
|
||||
is_neox_style: bool):
|
||||
num_tokens = positions.numel()
|
||||
cos, sin = cos_sin_cache.chunk(2, dim=-1)
|
||||
query_shape = query.shape
|
||||
key_shape = key.shape
|
||||
rotate_style = 0 if is_neox_style else 1
|
||||
|
||||
query = query.view(num_tokens, -1, head_size)
|
||||
key = key.view(num_tokens, -1, head_size)
|
||||
query_ = query[..., :rotary_dim]
|
||||
key_ = key[..., :rotary_dim]
|
||||
positions = positions.view(*query.shape[:1])
|
||||
torch.ops.vllm.rocm_aiter_rotary_emb_with_key_forward_triton(
|
||||
positions,
|
||||
sin,
|
||||
cos,
|
||||
query_,
|
||||
key_,
|
||||
rotate_style,
|
||||
False,
|
||||
)
|
||||
query = query.view(query_shape)
|
||||
key = key.view(key_shape)
|
||||
Reference in New Issue
Block a user