[Bugfix] Missing NIXL metadata for handshake initialization if instance spans multi-node (#26338)
Signed-off-by: Guan Luo <gluo@nvidia.com> Signed-off-by: GuanLuo <41310872+GuanLuo@users.noreply.github.com> Signed-off-by: Guan Luo <41310872+GuanLuo@users.noreply.github.com> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
This commit is contained in:
co-authored by
Nicolò Lucchesi
parent
7e06c40e63
commit
d6517be3cd
@@ -9,6 +9,9 @@ from typing import TYPE_CHECKING, Literal, TypeVar, overload
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import KVOutputAggregator
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorHandshakeMetadata,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.tasks import SupportedTask
|
||||
@@ -177,6 +180,11 @@ class Executor(ABC):
|
||||
):
|
||||
raise NotImplementedError
|
||||
|
||||
def get_kv_connector_handshake_metadata(
|
||||
self,
|
||||
) -> list[dict[int, KVConnectorHandshakeMetadata]]:
|
||||
return self.collective_rpc("get_kv_connector_handshake_metadata")
|
||||
|
||||
@overload
|
||||
def execute_model(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user