[P/D] NIXL Integration (#17751)
Signed-off-by: ApostaC <yihua98@uchicago.edu> Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com> Signed-off-by: rshaw@neuralmagic.com <robertgshaw2@gmail.com> Signed-off-by: Robert Shaw <rshaw@neuralmagic.com> Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Nick Hill <nhill@redhat.com> Signed-off-by: Brent Salisbury <bsalisbu@redhat.com> Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com> Co-authored-by: ApostaC <yihua98@uchicago.edu> Co-authored-by: Robert Shaw <rshaw@neuralmagic.com> Co-authored-by: mgoin <mgoin64@gmail.com> Co-authored-by: Nick Hill <nhill@redhat.com> Co-authored-by: Tyler Michael Smith <tysmith@redhat.com> Co-authored-by: Brent Salisbury <bsalisbu@redhat.com>
This commit is contained in:
co-authored by
Tyler Michael Smith
ApostaC
Robert Shaw
mgoin
Nick Hill
Tyler Michael Smith
Brent Salisbury
parent
05a4324f8e
commit
d19110204c
@@ -11,10 +11,6 @@ import torch.distributed as dist
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer import (get_kv_transfer_group,
|
||||
has_kv_transfer_group,
|
||||
is_v1_kv_transfer_group)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1 import KVConnectorBase_V1
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -106,16 +102,6 @@ def set_forward_context(attn_metadata: Any,
|
||||
attn_metadata=attn_metadata,
|
||||
dp_metadata=dp_metadata)
|
||||
|
||||
# KVConnector: trigger (possibly async) load before forward.
|
||||
# Each attn layer will block until the reading is complete.
|
||||
trigger_kv_transfer = (attn_metadata is not None
|
||||
and has_kv_transfer_group()
|
||||
and is_v1_kv_transfer_group())
|
||||
if trigger_kv_transfer:
|
||||
kv_connector = get_kv_transfer_group()
|
||||
assert isinstance(kv_connector, KVConnectorBase_V1)
|
||||
kv_connector.start_load_kv(_forward_context)
|
||||
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
@@ -152,11 +138,4 @@ def set_forward_context(attn_metadata: Any,
|
||||
"(batchsize, count, median_time(ms)): %s"),
|
||||
forward_stats)
|
||||
|
||||
# KVConnector: each attn layer triggers (possibly async) save.
|
||||
# Ensure all those operations complete before forward() is done.
|
||||
if trigger_kv_transfer:
|
||||
kv_connector = get_kv_transfer_group()
|
||||
assert isinstance(kv_connector, KVConnectorBase_V1)
|
||||
kv_connector.wait_for_save()
|
||||
|
||||
_forward_context = prev_context
|
||||
|
||||
Reference in New Issue
Block a user