[Bugfix][PD] Fix multi-node TP (TP>8) (#39907)
Signed-off-by: NickLucche <nlucches@redhat.com>
This commit is contained in:
@@ -48,6 +48,20 @@ def is_v1_kv_transfer_group(connector: KVConnectorBaseType | None = None) -> boo
|
||||
return isinstance(connector, KVConnectorBase_V1)
|
||||
|
||||
|
||||
def _sync_engine_id_across_tp(vllm_config: "VllmConfig") -> None:
|
||||
"""Broadcast engine_id from TP rank 0 so all workers in a
|
||||
multi-node TP group share the same value."""
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_tp_group,
|
||||
)
|
||||
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
synced_id = get_tp_group().broadcast_object(
|
||||
vllm_config.kv_transfer_config.engine_id, src=0
|
||||
)
|
||||
vllm_config.kv_transfer_config.engine_id = synced_id
|
||||
|
||||
|
||||
def ensure_kv_transfer_initialized(
|
||||
vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig"
|
||||
) -> None:
|
||||
@@ -64,6 +78,8 @@ def ensure_kv_transfer_initialized(
|
||||
vllm_config.kv_transfer_config.is_kv_transfer_instance
|
||||
and _KV_CONNECTOR_AGENT is None
|
||||
):
|
||||
_sync_engine_id_across_tp(vllm_config)
|
||||
|
||||
_KV_CONNECTOR_AGENT = KVConnectorFactory.create_connector(
|
||||
config=vllm_config,
|
||||
role=KVConnectorRole.WORKER,
|
||||
|
||||
Reference in New Issue
Block a user