[Attention Backend] add HPC-Ops Attention backend (#46020)
Signed-off-by: chengvjiang <chengvjiang@tencent.com> Co-authored-by: chengvjiang <chengvjiang@tencent.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
chengvjiang
Andreas Karatzas
parent
8e9d70fdd5
commit
8cf7c4d8ad
@@ -458,6 +458,7 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
# shape does not match the query shape, so we optionally let the model
|
||||
# definition specify the output tensor shape.
|
||||
output_shape: torch.Size | None = None,
|
||||
output_dtype: torch.dtype | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
The KV cache is stored inside this class and is accessed via
|
||||
@@ -472,7 +473,8 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
torch.ops.vllm.maybe_calc_kv_scales(
|
||||
query, key, value, _encode_layer_name(self.layer_name)
|
||||
)
|
||||
output_dtype = query.dtype
|
||||
if output_dtype is None:
|
||||
output_dtype = query.dtype
|
||||
if self.query_quant is not None:
|
||||
# quantizing with a simple torch operation enables
|
||||
# torch.compile to fuse this into previous ops
|
||||
|
||||
Reference in New Issue
Block a user