[Perf][PCP] Shard decode requests across PCP ranks (#52162)
Signed-off-by: QiuChunshuo <qiuchunshuo@huawei.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Lucas Wilkinson
parent
a810316c08
commit
8486f1ebfe
@@ -599,6 +599,19 @@ class ParallelConfig:
|
||||
"""Process world size across TP, PCP, PP, and DP."""
|
||||
return self.world_size * self.data_parallel_size
|
||||
|
||||
@property
|
||||
def pcp_shard_decode_requests(self) -> bool:
|
||||
"""Whether PCP can shard decode requests across its ranks.
|
||||
|
||||
PCP-only execution replicates the KV cache, so each decode request can
|
||||
have a single PCP owner. DCP shards the KV cache and therefore requires
|
||||
every decode request to run on every participating DCP rank.
|
||||
"""
|
||||
return (
|
||||
self.prefill_context_parallel_size > 1
|
||||
and self.decode_context_parallel_size == 1
|
||||
)
|
||||
|
||||
@property
|
||||
def use_ubatching(self) -> bool:
|
||||
return self.enable_dbo or self.ubatch_size > 1
|
||||
|
||||
Reference in New Issue
Block a user