[Perf][PCP] Shard decode requests across PCP ranks (#52162)

Signed-off-by: QiuChunshuo <qiuchunshuo@huawei.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Qiu Chunshuo
2026-10-01 23:50:13 -04:00
committed by GitHub
co-authored by Lucas Wilkinson
parent a810316c08
commit 8486f1ebfe
13 changed files with 406 additions and 33 deletions
@@ -335,6 +335,7 @@ def sparse_attn_indexer(
topk_indices_buffer: torch.Tensor,
skip_k_cache_insert: bool,
use_pcp: bool,
pcp_shard_decode_requests: bool,
dense_mha_metadata_layer_name: LayerNameType,
use_fp4_cache: bool = False,
dcp_rank: int = 0,
@@ -404,6 +405,7 @@ def sparse_attn_indexer(
topk_indices_buffer,
skip_k_cache_insert,
use_pcp,
pcp_shard_decode_requests,
dense_mha_metadata_layer_name,
use_fp4_cache,
candidate_blocks=candidate_blocks,
@@ -442,6 +444,7 @@ def sparse_attn_indexer(
slot_mapping,
num_decode_tokens,
use_pcp,
pcp_shard_decode_requests=pcp_shard_decode_requests,
)
# scale_fmt can be None, but the function expects str
assert scale_fmt is not None
@@ -817,6 +820,7 @@ def sparse_attn_indexer_fake(
topk_indices_buffer: torch.Tensor | None,
skip_k_cache_insert: bool,
use_pcp: bool,
pcp_shard_decode_requests: bool,
dense_mha_metadata_layer_name: LayerNameType,
use_fp4_cache: bool = False,
dcp_rank: int = 0,
@@ -898,6 +902,7 @@ class SparseAttnIndexer(CustomOp):
self.dcp_world_size = parallel_config.decode_context_parallel_size
self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_world_size > 1 else 0
self.use_pcp = parallel_config.prefill_context_parallel_size > 1
self.pcp_shard_decode_requests = parallel_config.pcp_shard_decode_requests
self._cp_kv_cache_interleave_size: int | None = None
if current_platform.is_cuda() and not has_deep_gemm():
raise RuntimeError(
@@ -994,6 +999,7 @@ class SparseAttnIndexer(CustomOp):
self.topk_indices_buffer,
self.skip_k_cache_insert,
self.use_pcp,
self.pcp_shard_decode_requests,
_encode_layer_name(self.dense_mha_metadata_layer_name),
self.use_fp4_cache,
self.dcp_rank,