Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
195 lines
6.6 KiB
Python
195 lines
6.6 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
from typing import TYPE_CHECKING, Any
|
|
|
|
import torch
|
|
|
|
from vllm.config import VllmConfig
|
|
from vllm.distributed.kv_transfer import (
|
|
get_kv_transfer_group,
|
|
has_kv_transfer_group,
|
|
kv_transfer_state,
|
|
)
|
|
from vllm.distributed.kv_transfer.kv_connector.utils import copy_kv_blocks
|
|
from vllm.forward_context import (
|
|
get_forward_context,
|
|
is_forward_context_available,
|
|
set_forward_context,
|
|
)
|
|
from vllm.v1.outputs import (
|
|
EMPTY_MODEL_RUNNER_OUTPUT,
|
|
KVConnectorOutput,
|
|
ModelRunnerOutput,
|
|
)
|
|
|
|
if TYPE_CHECKING:
|
|
from vllm.v1.core.sched.output import SchedulerOutput
|
|
from vllm.v1.worker.gpu.input_batch import InputBatch
|
|
|
|
|
|
class KVConnector:
|
|
"""KVConnector interface used by GPUModelRunner."""
|
|
|
|
def pre_forward(
|
|
self,
|
|
scheduler_output: "SchedulerOutput",
|
|
input_batch: "InputBatch | None" = None,
|
|
attn_metadata: dict[str, Any] | None = None,
|
|
) -> None:
|
|
pass
|
|
|
|
def finish_forward(self) -> None:
|
|
pass
|
|
|
|
def post_forward(
|
|
self, finished_req_ids: set[str], defer_finalize: bool = False
|
|
) -> KVConnectorOutput | None:
|
|
return None
|
|
|
|
def finalize_step(self) -> None:
|
|
pass
|
|
|
|
def no_forward(self, scheduler_output: "SchedulerOutput") -> ModelRunnerOutput:
|
|
return EMPTY_MODEL_RUNNER_OUTPUT
|
|
|
|
def set_disabled(self, disabled: bool) -> None:
|
|
pass
|
|
|
|
def reset_capture_state(self) -> None:
|
|
pass
|
|
|
|
|
|
class ActiveKVConnector(KVConnector):
|
|
def __init__(
|
|
self, vllm_config: VllmConfig, kv_caches_dict: dict[str, torch.Tensor]
|
|
):
|
|
self.vllm_config = vllm_config
|
|
self.kv_connector = get_kv_transfer_group()
|
|
# Register kv caches with KV Connector if applicable.
|
|
self.kv_connector.register_kv_caches(kv_caches_dict)
|
|
self.kv_connector.set_host_xfer_buffer_ops(copy_kv_blocks)
|
|
|
|
self._pending_load_kwargs: dict[str, Any] | None = None
|
|
self._disabled = False
|
|
|
|
def pre_forward(
|
|
self,
|
|
scheduler_output: "SchedulerOutput",
|
|
input_batch: "InputBatch | None" = None,
|
|
attn_metadata: dict[str, Any] | None = None,
|
|
) -> None:
|
|
if self._disabled:
|
|
return
|
|
|
|
kv_connector_metadata = scheduler_output.kv_connector_metadata
|
|
assert kv_connector_metadata is not None
|
|
self.kv_connector.handle_preemptions(kv_connector_metadata)
|
|
self.kv_connector.bind_connector_metadata(kv_connector_metadata)
|
|
self._pending_load_kwargs = self._build_load_kwargs(
|
|
scheduler_output, input_batch, attn_metadata
|
|
)
|
|
|
|
if scheduler_output.has_sync_kv_loads:
|
|
# Sync loads need to run before this step's forward.
|
|
self._start_load_kv()
|
|
# Otherwise start the async load after forward to keep submission
|
|
# off the critical path.
|
|
|
|
@staticmethod
|
|
def _build_load_kwargs(
|
|
scheduler_output: "SchedulerOutput",
|
|
input_batch: "InputBatch | None",
|
|
attn_metadata: dict[str, Any] | None,
|
|
) -> dict[str, Any]:
|
|
"""Keyword args for the connector's start_load_kv()."""
|
|
if input_batch is None:
|
|
# No forward this step.
|
|
return {}
|
|
load_kwargs: dict[str, Any] = dict(
|
|
scheduler_output=scheduler_output,
|
|
request_state_indices=input_batch.idx_mapping,
|
|
request_ids=input_batch.req_ids,
|
|
num_tokens=input_batch.num_tokens,
|
|
)
|
|
if attn_metadata is not None:
|
|
load_kwargs["attn_metadata"] = attn_metadata
|
|
return load_kwargs
|
|
|
|
def _start_load_kv(self) -> None:
|
|
load_kwargs = self._pending_load_kwargs
|
|
assert load_kwargs is not None
|
|
self._pending_load_kwargs = None
|
|
# TODO: sort out KV Connectors' use of forward_context
|
|
if is_forward_context_available():
|
|
self.kv_connector.start_load_kv(get_forward_context(), **load_kwargs)
|
|
else:
|
|
with set_forward_context(None, self.vllm_config):
|
|
self.kv_connector.start_load_kv(get_forward_context(), **load_kwargs)
|
|
|
|
def finish_forward(self) -> None:
|
|
if not self._disabled:
|
|
self.kv_connector.finish_forward()
|
|
|
|
def reset_capture_state(self) -> None:
|
|
self.kv_connector.reset_capture_state()
|
|
|
|
def post_forward(
|
|
self, finished_req_ids: set[str], defer_finalize: bool = False
|
|
) -> KVConnectorOutput | None:
|
|
if self._disabled:
|
|
return None
|
|
|
|
if self._pending_load_kwargs is not None:
|
|
self._start_load_kv()
|
|
|
|
output = KVConnectorOutput()
|
|
if not defer_finalize:
|
|
self.kv_connector.wait_for_save()
|
|
transfer_results = self.kv_connector.get_transfer_results(finished_req_ids)
|
|
output.finished_sending = transfer_results.finished_sending or None
|
|
output.finished_recving = transfer_results.finished_recving or None
|
|
output.failed_recving = transfer_results.failed_recving
|
|
output.invalid_block_ids = self.kv_connector.get_block_ids_with_load_errors()
|
|
output.kv_connector_stats = self.kv_connector.get_kv_connector_stats()
|
|
output.kv_cache_events = self.kv_connector.get_kv_connector_kv_cache_events()
|
|
output.kv_connector_worker_meta = (
|
|
self.kv_connector.build_connector_worker_meta()
|
|
)
|
|
if not defer_finalize:
|
|
self.kv_connector.clear_connector_metadata()
|
|
return output
|
|
|
|
def finalize_step(self) -> None:
|
|
if self._disabled:
|
|
return
|
|
self.kv_connector.wait_for_save()
|
|
self.kv_connector.clear_connector_metadata()
|
|
|
|
def no_forward(self, scheduler_output: "SchedulerOutput") -> ModelRunnerOutput:
|
|
if self._disabled:
|
|
return EMPTY_MODEL_RUNNER_OUTPUT
|
|
|
|
self.pre_forward(scheduler_output)
|
|
self.finish_forward()
|
|
finished_req_ids = scheduler_output.finished_req_ids
|
|
kv_connector_output = self.post_forward(finished_req_ids)
|
|
return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
|
|
|
|
def set_disabled(self, disabled: bool) -> None:
|
|
# Ensure that layer-wise connector hooks aren't called when disabled.
|
|
kv_transfer_state._KV_CONNECTOR_AGENT = None if disabled else self.kv_connector
|
|
self._disabled = disabled
|
|
|
|
|
|
NO_OP_KV_CONNECTOR = KVConnector()
|
|
|
|
|
|
def get_kv_connector(
|
|
vllm_config: VllmConfig, kv_caches_dict: dict[str, torch.Tensor]
|
|
) -> KVConnector:
|
|
if not has_kv_transfer_group():
|
|
# No-op connector.
|
|
return NO_OP_KV_CONNECTOR
|
|
|
|
return ActiveKVConnector(vllm_config, kv_caches_dict)
|