[Attention Backend] add HPC-Ops Attention backend (#46020)

Signed-off-by: chengvjiang <chengvjiang@tencent.com>
Co-authored-by: chengvjiang <chengvjiang@tencent.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Cheng Jiang
2026-06-30 18:17:43 +08:00
committed by GitHub
co-authored by chengvjiang Andreas Karatzas
parent 8e9d70fdd5
commit 8cf7c4d8ad
10 changed files with 978 additions and 14 deletions
@@ -458,6 +458,7 @@ class Attention(nn.Module, AttentionLayerBase):
# shape does not match the query shape, so we optionally let the model
# definition specify the output tensor shape.
output_shape: torch.Size | None = None,
output_dtype: torch.dtype | None = None,
) -> torch.Tensor:
"""
The KV cache is stored inside this class and is accessed via
@@ -472,7 +473,8 @@ class Attention(nn.Module, AttentionLayerBase):
torch.ops.vllm.maybe_calc_kv_scales(
query, key, value, _encode_layer_name(self.layer_name)
)
output_dtype = query.dtype
if output_dtype is None:
output_dtype = query.dtype
if self.query_quant is not None:
# quantizing with a simple torch operation enables
# torch.compile to fuse this into previous ops