[nv] add pcp support in dsv3.2 (#52046)
Signed-off-by: Summer Yang <girasoleyang@gmail.com>
This commit is contained in:
@@ -299,7 +299,7 @@ def sparse_attn_indexer(
|
||||
kv_cache: torch.Tensor,
|
||||
q_quant: torch.Tensor,
|
||||
q_scale: torch.Tensor | None,
|
||||
k: torch.Tensor,
|
||||
k: torch.Tensor | None,
|
||||
weights: torch.Tensor,
|
||||
quant_block_size: int,
|
||||
scale_fmt: str | None,
|
||||
@@ -695,7 +695,7 @@ def sparse_attn_indexer_fake(
|
||||
kv_cache: torch.Tensor,
|
||||
q_quant: torch.Tensor,
|
||||
q_scale: torch.Tensor | None,
|
||||
k: torch.Tensor,
|
||||
k: torch.Tensor | None,
|
||||
weights: torch.Tensor,
|
||||
quant_block_size: int,
|
||||
scale_fmt: str | None,
|
||||
@@ -781,7 +781,7 @@ class SparseAttnIndexer(CustomOp):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
q_quant: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
k: torch.Tensor,
|
||||
k: torch.Tensor | None,
|
||||
weights: torch.Tensor,
|
||||
):
|
||||
if current_platform.is_cuda() or current_platform.is_xpu():
|
||||
@@ -798,7 +798,7 @@ class SparseAttnIndexer(CustomOp):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
q_quant: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
k: torch.Tensor,
|
||||
k: torch.Tensor | None,
|
||||
weights: torch.Tensor,
|
||||
):
|
||||
# FP8 path: single tensor (per-token scale is folded into `weights`).
|
||||
@@ -835,7 +835,7 @@ class SparseAttnIndexer(CustomOp):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
q_fp8: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
k: torch.Tensor | None,
|
||||
weights: torch.Tensor,
|
||||
):
|
||||
return self.forward_cuda(hidden_states, q_fp8, k, weights)
|
||||
@@ -844,7 +844,7 @@ class SparseAttnIndexer(CustomOp):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
q_quant: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
k: torch.Tensor,
|
||||
k: torch.Tensor | None,
|
||||
weights: torch.Tensor,
|
||||
):
|
||||
assert not self.use_fp4_cache, "AMD platform doesn't support fp4 cache yet"
|
||||
|
||||
Reference in New Issue
Block a user