[Perf][PCP] Shard decode requests across PCP ranks (#52162)
Signed-off-by: QiuChunshuo <qiuchunshuo@huawei.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Lucas Wilkinson
parent
a810316c08
commit
8486f1ebfe
@@ -609,6 +609,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
vllm_config = get_current_vllm_config()
|
||||
parallel_config = vllm_config.parallel_config
|
||||
self.use_pcp = parallel_config.prefill_context_parallel_size > 1
|
||||
self.pcp_shard_decode_requests = parallel_config.pcp_shard_decode_requests
|
||||
compilation_config = vllm_config.compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
@@ -784,6 +785,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
slot_mapping,
|
||||
attn_metadata.num_decode_tokens if attn_metadata is not None else None,
|
||||
self.use_pcp,
|
||||
pcp_shard_decode_requests=self.pcp_shard_decode_requests,
|
||||
)
|
||||
assert slot_mapping is not None
|
||||
if cache is not None:
|
||||
|
||||
Reference in New Issue
Block a user