[Hardware][Neuron] Refactor neuron support (#3471)

This commit is contained in:
Zhuohan Li
2024-03-22 01:22:17 +00:00
committed by GitHub
parent ea5f14e6ff
commit e90fc21f2e
33 changed files with 615 additions and 549 deletions
+4 -3
View File
@@ -1,8 +1,9 @@
from dataclasses import dataclass, fields
from typing import Optional, List, Any, Dict
from typing import TYPE_CHECKING, Optional, List, Any, Dict
import torch
from xformers.ops.fmha.attn_bias import AttentionBias
if TYPE_CHECKING:
from xformers.ops.fmha.attn_bias import AttentionBias
@dataclass
@@ -82,7 +83,7 @@ class InputMetadata:
# when alibi slopes is used. It is because of the limitation
# from xformer API.
# will not appear in the __repr__ and __init__
self.attn_bias: Optional[List[AttentionBias]] = None
self.attn_bias: Optional[List["AttentionBias"]] = None
# Cuda graph is only used for decoding now.
if self.use_cuda_graph: