[Perf][PCP] Shard decode requests across PCP ranks (#52162)
Signed-off-by: QiuChunshuo <qiuchunshuo@huawei.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Lucas Wilkinson
parent
a810316c08
commit
8486f1ebfe
@@ -609,6 +609,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
vllm_config = get_current_vllm_config()
|
||||
parallel_config = vllm_config.parallel_config
|
||||
self.use_pcp = parallel_config.prefill_context_parallel_size > 1
|
||||
self.pcp_shard_decode_requests = parallel_config.pcp_shard_decode_requests
|
||||
compilation_config = vllm_config.compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
@@ -784,6 +785,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
slot_mapping,
|
||||
attn_metadata.num_decode_tokens if attn_metadata is not None else None,
|
||||
self.use_pcp,
|
||||
pcp_shard_decode_requests=self.pcp_shard_decode_requests,
|
||||
)
|
||||
assert slot_mapping is not None
|
||||
if cache is not None:
|
||||
|
||||
@@ -335,6 +335,7 @@ def sparse_attn_indexer(
|
||||
topk_indices_buffer: torch.Tensor,
|
||||
skip_k_cache_insert: bool,
|
||||
use_pcp: bool,
|
||||
pcp_shard_decode_requests: bool,
|
||||
dense_mha_metadata_layer_name: LayerNameType,
|
||||
use_fp4_cache: bool = False,
|
||||
dcp_rank: int = 0,
|
||||
@@ -404,6 +405,7 @@ def sparse_attn_indexer(
|
||||
topk_indices_buffer,
|
||||
skip_k_cache_insert,
|
||||
use_pcp,
|
||||
pcp_shard_decode_requests,
|
||||
dense_mha_metadata_layer_name,
|
||||
use_fp4_cache,
|
||||
candidate_blocks=candidate_blocks,
|
||||
@@ -442,6 +444,7 @@ def sparse_attn_indexer(
|
||||
slot_mapping,
|
||||
num_decode_tokens,
|
||||
use_pcp,
|
||||
pcp_shard_decode_requests=pcp_shard_decode_requests,
|
||||
)
|
||||
# scale_fmt can be None, but the function expects str
|
||||
assert scale_fmt is not None
|
||||
@@ -817,6 +820,7 @@ def sparse_attn_indexer_fake(
|
||||
topk_indices_buffer: torch.Tensor | None,
|
||||
skip_k_cache_insert: bool,
|
||||
use_pcp: bool,
|
||||
pcp_shard_decode_requests: bool,
|
||||
dense_mha_metadata_layer_name: LayerNameType,
|
||||
use_fp4_cache: bool = False,
|
||||
dcp_rank: int = 0,
|
||||
@@ -898,6 +902,7 @@ class SparseAttnIndexer(CustomOp):
|
||||
self.dcp_world_size = parallel_config.decode_context_parallel_size
|
||||
self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_world_size > 1 else 0
|
||||
self.use_pcp = parallel_config.prefill_context_parallel_size > 1
|
||||
self.pcp_shard_decode_requests = parallel_config.pcp_shard_decode_requests
|
||||
self._cp_kv_cache_interleave_size: int | None = None
|
||||
if current_platform.is_cuda() and not has_deep_gemm():
|
||||
raise RuntimeError(
|
||||
@@ -994,6 +999,7 @@ class SparseAttnIndexer(CustomOp):
|
||||
self.topk_indices_buffer,
|
||||
self.skip_k_cache_insert,
|
||||
self.use_pcp,
|
||||
self.pcp_shard_decode_requests,
|
||||
_encode_layer_name(self.dense_mha_metadata_layer_name),
|
||||
self.use_fp4_cache,
|
||||
self.dcp_rank,
|
||||
|
||||
Reference in New Issue
Block a user