[Neuron] Add custom_ops for neuron backend (#13246)
Signed-off-by: Liangfu Chen <liangfc@amazon.com> Co-authored-by: George Novack <gnovack@amazon.com> Co-authored-by: Aoyu Zhang <aoyuzhan@amazon.com>
This commit is contained in:
co-authored by
George Novack
Aoyu Zhang
parent
340e39e387
commit
f75aa72732
@@ -59,6 +59,11 @@ class CustomOp(nn.Module):
|
||||
# PyTorch-native implementation.
|
||||
return self.forward_native(*args, **kwargs)
|
||||
|
||||
def forward_neuron(self, *args, **kwargs):
|
||||
# By default, we assume that Neuron ops are compatible with the
|
||||
# PyTorch-native implementation.
|
||||
return self.forward_native(*args, **kwargs)
|
||||
|
||||
def forward_oot(self, *args, **kwargs):
|
||||
# By default, we assume that OOT ops are compatible with the
|
||||
# PyTorch-native implementation.
|
||||
@@ -88,6 +93,8 @@ class CustomOp(nn.Module):
|
||||
return self.forward_tpu
|
||||
elif current_platform.is_xpu():
|
||||
return self.forward_xpu
|
||||
elif current_platform.is_neuron():
|
||||
return self.forward_neuron
|
||||
elif current_platform.is_out_of_tree():
|
||||
return self.forward_oot
|
||||
else:
|
||||
|
||||
@@ -89,6 +89,13 @@ class SiluAndMul(CustomOp):
|
||||
self.op(out, x)
|
||||
return out
|
||||
|
||||
def forward_neuron(self, x: torch.Tensor) -> torch.Tensor:
|
||||
d = x.shape[-1] // 2
|
||||
x_reshaped = x.view(-1, x.shape[-1])
|
||||
s = x_reshaped[:, :d] * F.sigmoid(x_reshaped[:, :d])
|
||||
result = s * x_reshaped[:, d:]
|
||||
return result.view(*x.shape[:-1], d)
|
||||
|
||||
|
||||
@CustomOp.register("mul_and_silu")
|
||||
class MulAndSilu(CustomOp):
|
||||
|
||||
@@ -53,6 +53,7 @@ class LogitsProcessor(nn.Module):
|
||||
# Whether to use gather or all-gather to gather the logits.
|
||||
parallel_config = get_current_vllm_config().parallel_config
|
||||
self.use_all_gather = current_platform.is_tpu() \
|
||||
or current_platform.is_neuron() \
|
||||
or envs.VLLM_USE_V1 \
|
||||
or parallel_config.distributed_executor_backend == "external_launcher" # noqa
|
||||
|
||||
|
||||
@@ -254,6 +254,82 @@ class RotaryEmbedding(CustomOp):
|
||||
key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape)
|
||||
return query, key
|
||||
|
||||
def forward_neuron(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
offsets: Optional[torch.Tensor] = None,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
|
||||
def _apply_rotary_emb_neuron(
|
||||
x: torch.Tensor,
|
||||
cos: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
is_neox_style: bool,
|
||||
) -> torch.Tensor:
|
||||
cos = cos.unsqueeze(-2).to(x.dtype)
|
||||
sin = sin.unsqueeze(-2).to(x.dtype)
|
||||
if is_neox_style:
|
||||
x1, x2 = torch.chunk(x, 2, dim=-1)
|
||||
else:
|
||||
# x1 = x[..., ::2]
|
||||
|
||||
# x2 = x[..., 1::2]
|
||||
d = x.shape[-1] // 2
|
||||
x_reshaped = x.view(-1, x.shape[-1])
|
||||
x1 = x_reshaped[:, ::2].view(*x.shape[:-1], d)
|
||||
x2 = x_reshaped[:, 1::2].view(*x.shape[:-1], d)
|
||||
o1 = x1 * cos - x2 * sin
|
||||
o2 = x2 * cos + x1 * sin
|
||||
if is_neox_style:
|
||||
return torch.cat((o1, o2), dim=-1)
|
||||
else:
|
||||
return torch.stack((o1, o2), dim=-1).flatten(-2)
|
||||
|
||||
if offsets is not None:
|
||||
positions = positions + offsets
|
||||
|
||||
self.cos_sin_cache = self.cos_sin_cache.to(query.device,
|
||||
dtype=query.dtype)
|
||||
|
||||
positions = positions.flatten()
|
||||
num_tokens = positions.shape[0]
|
||||
cos_sin = self.cos_sin_cache.index_select(0, positions)
|
||||
cos, sin = cos_sin.chunk(2, dim=-1)
|
||||
|
||||
query_shape = query.shape
|
||||
query = query.view(num_tokens, -1, self.head_size)
|
||||
key_shape = key.shape
|
||||
key = key.view(num_tokens, -1, self.head_size)
|
||||
|
||||
if self.rotary_dim == self.head_size:
|
||||
query = _apply_rotary_emb(query, cos, sin, self.is_neox_style)
|
||||
query = query.reshape(query_shape)
|
||||
key = _apply_rotary_emb(key, cos, sin, self.is_neox_style)
|
||||
key = key.reshape(key_shape)
|
||||
else:
|
||||
head_size = query.shape[-1]
|
||||
query_reshaped = query.view(-1, head_size)
|
||||
query_pass = query_reshaped[:, self.rotary_dim:].view(
|
||||
*query.shape[:-1], head_size - self.rotary_dim)
|
||||
query_rot = query_reshaped[:, :self.rotary_dim].view(
|
||||
*query.shape[:-1], self.rotary_dim)
|
||||
query_rot = _apply_rotary_emb_neuron(query_rot, cos, sin,
|
||||
self.is_neox_style)
|
||||
query = torch.cat((query_rot, query_pass),
|
||||
dim=-1).reshape(query_shape)
|
||||
|
||||
key_reshaped = key.view(-1, head_size)
|
||||
key_pass = key_reshaped[:, self.rotary_dim:].view(
|
||||
*key.shape[:-1], head_size - self.rotary_dim)
|
||||
key_rot = key_reshaped[:, :self.rotary_dim].view(
|
||||
*key.shape[:-1], self.rotary_dim)
|
||||
key_rot = _apply_rotary_emb_neuron(key_rot, cos, sin,
|
||||
self.is_neox_style)
|
||||
key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape)
|
||||
return query, key
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
s = f"head_size={self.head_size}, rotary_dim={self.rotary_dim}"
|
||||
s += f", max_position_embeddings={self.max_position_embeddings}"
|
||||
|
||||
Reference in New Issue
Block a user