[ROCm][Connector] SWA+HMA-support in MoRI-IO connector (Gemma4) (#53721)
Signed-off-by: simondanielsson <simon.danielsson99@hotmail.com>
This commit is contained in:
@@ -39,10 +39,12 @@ from vllm.utils.network_utils import (
|
||||
make_zmq_path,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
ChunkedLocalAttentionSpec,
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
SlidingWindowSpec,
|
||||
compute_layer_kv_cache_shape_bytes,
|
||||
)
|
||||
|
||||
@@ -338,7 +340,7 @@ def test_write_mode_saves_local_block_ids():
|
||||
req_meta = kv_connector_metadata.reqs_to_save[request_id]
|
||||
|
||||
for block_id, block in zip(
|
||||
req_meta.local_block_ids,
|
||||
req_meta.local_block_ids[0],
|
||||
scheduler.kv_cache_manager.coordinator.single_type_managers[0].req_to_blocks[
|
||||
request_id
|
||||
],
|
||||
@@ -393,7 +395,7 @@ def test_write_mode_with_chunked_prefill_saves_local_block_ids():
|
||||
req_meta = kv_connector_metadata.reqs_to_save[request_id]
|
||||
|
||||
for block_id, block in zip(
|
||||
req_meta.local_block_ids,
|
||||
req_meta.local_block_ids[0],
|
||||
scheduler.kv_cache_manager.coordinator.single_type_managers[0].req_to_blocks[
|
||||
request_id
|
||||
],
|
||||
@@ -432,7 +434,7 @@ def test_read_mode_loads_remote_block_ids():
|
||||
].req_to_blocks[request_id]
|
||||
|
||||
# Set remote block ids to be fetched.
|
||||
request.kv_transfer_params["remote_block_ids"] = block_list
|
||||
request.kv_transfer_params["remote_block_ids"] = [block_list]
|
||||
|
||||
# Remote Prefill, triggers MoRIIOConnectorMetadata.
|
||||
|
||||
@@ -457,7 +459,7 @@ def test_read_mode_loads_remote_block_ids():
|
||||
req_meta = kv_connector_metadata.reqs_to_recv[request_id]
|
||||
|
||||
for block_id, block in zip(
|
||||
req_meta.local_block_ids,
|
||||
req_meta.local_block_ids[0],
|
||||
scheduler.kv_cache_manager.coordinator.single_type_managers[0].req_to_blocks[
|
||||
request_id
|
||||
],
|
||||
@@ -699,3 +701,225 @@ def test_resolve_host_ip_prefers_extra_config():
|
||||
fallback = get_ip()
|
||||
assert resolve_host_ip({}) == fallback
|
||||
assert resolve_host_ip({"host_ip": ""}) == fallback
|
||||
|
||||
|
||||
def _make_hybrid_kv_cache_config() -> KVCacheConfig:
|
||||
"""One full-attention group + one sliding-window group (Gemma-like)."""
|
||||
full_spec = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=4, head_size=64, dtype=torch.float16
|
||||
)
|
||||
sw_spec = SlidingWindowSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=4,
|
||||
head_size=64,
|
||||
dtype=torch.float16,
|
||||
sliding_window=32,
|
||||
)
|
||||
num_blocks = 2
|
||||
page = full_spec.page_size_bytes
|
||||
return KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=2 * num_blocks * page,
|
||||
layers=["full0", "sw0"],
|
||||
layer_stride=num_blocks * page,
|
||||
block_stride=page,
|
||||
)
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(layer_names=["full0"], kv_cache_spec=full_spec),
|
||||
KVCacheGroupSpec(layer_names=["sw0"], kv_cache_spec=sw_spec),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _read_scheduler(
|
||||
kv_cache_config: KVCacheConfig, disable_hma: bool = False
|
||||
) -> MoRIIOConnectorScheduler:
|
||||
vllm_config = create_vllm_config(role="kv_producer", read_mode=True)
|
||||
vllm_config.scheduler_config.disable_hybrid_kv_cache_manager = disable_hma
|
||||
with set_current_vllm_config(vllm_config):
|
||||
connector = MoRIIOConnector(
|
||||
vllm_config, KVConnectorRole.SCHEDULER, kv_cache_config
|
||||
)
|
||||
assert connector.connector_scheduler is not None
|
||||
return connector.connector_scheduler
|
||||
|
||||
|
||||
def test_hma_blocks_per_sw_two_groups():
|
||||
"""A Full + SlidingWindow config runs with HMA and computes block budgets
|
||||
correctly"""
|
||||
scheduler = _read_scheduler(_make_hybrid_kv_cache_config())
|
||||
assert scheduler._is_hma_required is True
|
||||
# cdiv(32, 16) + 1 == 3 for the sliding-window group, 0 for full attention.
|
||||
assert scheduler.blocks_per_sw == [0, 3]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"swa_enabled, disable_hma, expected_is_hma",
|
||||
[
|
||||
(True, False, True), # sliding-window group present, HMA enabled
|
||||
(True, True, False), # sliding-window group present but HMA disabled
|
||||
(False, False, False), # full-attention only, HMA not needed
|
||||
],
|
||||
)
|
||||
def test_is_hma_required(swa_enabled, disable_hma, expected_is_hma):
|
||||
"""_is_hma_required tracks both the KV cache groups and the
|
||||
--disable-hybrid-kv-cache-manager flag. When HMA is off,
|
||||
get_exchange_clipped_blocks must be a no-op."""
|
||||
config = (
|
||||
_make_hybrid_kv_cache_config() if swa_enabled else _make_test_kv_cache_config()
|
||||
)
|
||||
scheduler = _read_scheduler(config, disable_hma=disable_hma)
|
||||
assert scheduler._is_hma_required is expected_is_hma
|
||||
if not expected_is_hma:
|
||||
blocks = [[1, 2, 3, 4, 5]]
|
||||
assert scheduler.get_exchange_clipped_blocks(blocks) == blocks
|
||||
|
||||
|
||||
def test_non_sliding_window_hybrid_is_rejected():
|
||||
"""A hybrid group that is not sliding-window (e.g. chunked-local
|
||||
attention) must fail closed rather than be silently mistransferred."""
|
||||
full_spec = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=4, head_size=64, dtype=torch.float16
|
||||
)
|
||||
local_spec = ChunkedLocalAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=4,
|
||||
head_size=64,
|
||||
dtype=torch.float16,
|
||||
attention_chunk_size=32,
|
||||
)
|
||||
num_blocks = 2
|
||||
page = full_spec.page_size_bytes
|
||||
config = KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=2 * num_blocks * page,
|
||||
layers=["full0", "local0"],
|
||||
layer_stride=num_blocks * page,
|
||||
block_stride=page,
|
||||
)
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(layer_names=["full0"], kv_cache_spec=full_spec),
|
||||
KVCacheGroupSpec(layer_names=["local0"], kv_cache_spec=local_spec),
|
||||
],
|
||||
)
|
||||
with pytest.raises(NotImplementedError, match="sliding-window hybrid"):
|
||||
_read_scheduler(config)
|
||||
|
||||
|
||||
def test_token_count_basis_uses_full_attention_group():
|
||||
"""Chunked-prefill token counting must use an unclipped full-attention
|
||||
group (blocks_per_sw == 0), not a clipped sliding-window group."""
|
||||
scheduler = _read_scheduler(_make_hybrid_kv_cache_config())
|
||||
# Group 0 is the full-attention group
|
||||
assert scheduler._full_attn_group_idx == 0
|
||||
assert scheduler._full_attn_block_size == 16
|
||||
|
||||
|
||||
def test_get_exchange_clipped_blocks_clips_only_sw_group():
|
||||
"""get_exchange_clipped_blocks keeps the full attn group intact and clips the
|
||||
sliding-window group to its window tail."""
|
||||
scheduler = _read_scheduler(_make_hybrid_kv_cache_config())
|
||||
full = [10, 11, 12, 13, 14]
|
||||
sw = [20, 21, 22, 23, 24]
|
||||
clipped = scheduler.get_exchange_clipped_blocks([full, sw])
|
||||
assert clipped[0] == full
|
||||
assert clipped[1] == [22, 23, 24]
|
||||
|
||||
|
||||
def test_metadata_hma_block_ids_preserved_per_group():
|
||||
"""add_new_req stores per-group (BlockIds) block lists unchanged for both
|
||||
read and write, so the hybrid group structure is retained."""
|
||||
metadata = MoRIIOConnectorMetadata()
|
||||
|
||||
# Assume:
|
||||
# - Full-attention group (6 blocks) +
|
||||
# - sliding-window group already clipped to its window tail (3 blocks).
|
||||
fa_blocks = [0, 1, 2, 3, 4, 5]
|
||||
sw_blocks = [10, 11, 12]
|
||||
local_block_ids = [fa_blocks, sw_blocks]
|
||||
remote_block_ids = [[100, 101, 102, 103, 104, 105], [200, 201, 202]]
|
||||
|
||||
base_params = {
|
||||
"remote_engine_id": "remote-engine",
|
||||
"remote_host": "127.0.0.1",
|
||||
"remote_handshake_port": 6301,
|
||||
"remote_notify_port": 61005,
|
||||
"remote_block_ids": remote_block_ids,
|
||||
}
|
||||
|
||||
# Read mode: both local and remote ids stay per-group.
|
||||
metadata.add_new_req(
|
||||
request_id="recv-req",
|
||||
local_block_ids=local_block_ids,
|
||||
kv_transfer_params={**base_params, "transfer_id": "recv-req"},
|
||||
)
|
||||
recv_meta = metadata.reqs_to_recv["recv-req"]
|
||||
assert recv_meta.local_block_ids == [fa_blocks, sw_blocks]
|
||||
assert recv_meta.remote_block_ids == remote_block_ids
|
||||
|
||||
# Write mode: the decode peer allocates its own blocks, so #remote_block_ids may
|
||||
# be empty, Local group structure is kept.
|
||||
metadata.add_new_req(
|
||||
request_id="save-req",
|
||||
local_block_ids=local_block_ids,
|
||||
kv_transfer_params={
|
||||
**base_params,
|
||||
"transfer_id": "save-req",
|
||||
"remote_block_ids": [],
|
||||
},
|
||||
write_mode=True,
|
||||
)
|
||||
save_meta = metadata.reqs_to_save["save-req"]
|
||||
assert save_meta.local_block_ids == [fa_blocks, sw_blocks]
|
||||
assert save_meta.remote_block_ids == []
|
||||
|
||||
|
||||
def test_single_group_path_unchanged():
|
||||
"""A full attn config does not enable HMA and never clips blocks."""
|
||||
scheduler = _read_scheduler(_make_test_kv_cache_config())
|
||||
assert scheduler._is_hma_required is False
|
||||
assert scheduler.blocks_per_sw == [0]
|
||||
blocks = [[1, 2, 3, 4, 5]]
|
||||
assert scheduler.get_exchange_clipped_blocks(blocks) == blocks
|
||||
|
||||
|
||||
def test_hybrid_write_mode_rejected():
|
||||
"""Hybrid KV cache groups are unsupported in WRITE mode and fail closed."""
|
||||
vllm_config = create_vllm_config(role="kv_producer", read_mode=False)
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
pytest.raises(NotImplementedError),
|
||||
):
|
||||
MoRIIOConnector(
|
||||
vllm_config,
|
||||
KVConnectorRole.SCHEDULER,
|
||||
_make_hybrid_kv_cache_config(),
|
||||
)
|
||||
|
||||
|
||||
def test_worker_layer_to_group_routing(mock_parallel_groups):
|
||||
"""The worker maps every layer to its KV cache group correctly."""
|
||||
vllm_config = create_vllm_config(role="kv_consumer", read_mode=True)
|
||||
# Building the worker directly bypasses MoRIIOConnector._set_port_defaults,
|
||||
# so provide the ports manually.
|
||||
vllm_config.kv_transfer_config.kv_connector_extra_config.update(
|
||||
{
|
||||
"http_port": 12346,
|
||||
"handshake_port": 12347,
|
||||
"notify_port": 12348,
|
||||
}
|
||||
)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
worker = FakeMoRIIOConnectorWorker(
|
||||
vllm_config,
|
||||
"engine0",
|
||||
hand_shake_latency=0,
|
||||
kv_cache_config=_make_hybrid_kv_cache_config(),
|
||||
)
|
||||
assert worker.layer_to_group == {"full0": 0, "sw0": 1}
|
||||
|
||||
@@ -20,12 +20,15 @@ from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import
|
||||
from vllm.v1.request import RequestStatus
|
||||
|
||||
_request_finished = MoRIIOConnectorScheduler.request_finished
|
||||
_clip_blocks = MoRIIOConnectorScheduler.get_exchange_clipped_blocks
|
||||
|
||||
|
||||
def _producer(global_dp_rank: int, dp_size: int, dp_size_local: int):
|
||||
"""Minimal prefill-side (producer) scheduler stand-in for request_finished."""
|
||||
return SimpleNamespace(
|
||||
s = SimpleNamespace(
|
||||
is_producer=True,
|
||||
_is_hma_required=False,
|
||||
blocks_per_sw=[0],
|
||||
_global_dp_rank=global_dp_rank,
|
||||
engine_id="engine-abc",
|
||||
host_ip="10.0.0.1",
|
||||
@@ -39,6 +42,8 @@ def _producer(global_dp_rank: int, dp_size: int, dp_size_local: int):
|
||||
)
|
||||
),
|
||||
)
|
||||
s.get_exchange_clipped_blocks = _clip_blocks.__get__(s)
|
||||
return s
|
||||
|
||||
|
||||
def _req(
|
||||
|
||||
@@ -14,6 +14,7 @@ import torch
|
||||
import zmq
|
||||
|
||||
from vllm.config import KVTransferConfig, VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorMetadata,
|
||||
)
|
||||
@@ -444,8 +445,8 @@ class ReqMeta:
|
||||
"""Metadata for a single request."""
|
||||
|
||||
transfer_id: TransferId
|
||||
local_block_ids: list[int]
|
||||
remote_block_ids: list[int]
|
||||
local_block_ids: BlockIds
|
||||
remote_block_ids: BlockIds
|
||||
remote_host: str
|
||||
remote_port: int
|
||||
remote_handshake_port: int
|
||||
@@ -481,7 +482,7 @@ class MoRIIOConnectorMetadata(KVConnectorMetadata):
|
||||
def add_new_req(
|
||||
self,
|
||||
request_id: ReqId,
|
||||
local_block_ids: list[int],
|
||||
local_block_ids: BlockIds,
|
||||
kv_transfer_params: dict[str, Any],
|
||||
write_mode=False,
|
||||
):
|
||||
|
||||
@@ -18,10 +18,12 @@ import torch
|
||||
import zmq
|
||||
|
||||
from vllm.config import CUDAGraphMode, VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorMetadata,
|
||||
KVConnectorRole,
|
||||
SupportsHMA,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
|
||||
ROLE,
|
||||
@@ -67,19 +69,24 @@ from vllm.distributed.parallel_state import (
|
||||
)
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.network_utils import (
|
||||
make_zmq_path,
|
||||
make_zmq_socket,
|
||||
)
|
||||
from vllm.v1.attention.selector import get_attn_backend
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.request import RequestStatus
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -189,7 +196,7 @@ def resolve_moriio_transfer_ack(
|
||||
return transfer_id
|
||||
|
||||
|
||||
class MoRIIOConnector(KVConnectorBase_V1):
|
||||
class MoRIIOConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
@@ -225,7 +232,7 @@ class MoRIIOConnector(KVConnectorBase_V1):
|
||||
)
|
||||
if role == KVConnectorRole.SCHEDULER:
|
||||
self.connector_scheduler: MoRIIOConnectorScheduler | None = (
|
||||
MoRIIOConnectorScheduler(vllm_config, self.engine_id)
|
||||
MoRIIOConnectorScheduler(vllm_config, self.engine_id, kv_cache_config)
|
||||
)
|
||||
self.connector_worker: MoRIIOConnectorWorker | None = None
|
||||
elif role == KVConnectorRole.WORKER:
|
||||
@@ -283,6 +290,14 @@ class MoRIIOConnector(KVConnectorBase_V1):
|
||||
self,
|
||||
request: "Request",
|
||||
block_ids: list[int],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.request_finished(request, (block_ids,))
|
||||
|
||||
def request_finished_all_groups(
|
||||
self,
|
||||
request: "Request",
|
||||
block_ids: BlockIds,
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.request_finished(request, block_ids)
|
||||
@@ -368,7 +383,12 @@ class MoRIIOConnector(KVConnectorBase_V1):
|
||||
class MoRIIOConnectorScheduler:
|
||||
"""Implementation of Scheduler side methods"""
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, engine_id: str):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
engine_id: str,
|
||||
kv_cache_config: "KVCacheConfig",
|
||||
):
|
||||
self.vllm_config = vllm_config
|
||||
|
||||
assert vllm_config.kv_transfer_config is not None, (
|
||||
@@ -378,6 +398,66 @@ class MoRIIOConnectorScheduler:
|
||||
self.block_size = vllm_config.cache_config.block_size
|
||||
self.engine_id: EngineId = engine_id
|
||||
self.mode = get_moriio_mode(self.kv_transfer_config)
|
||||
|
||||
self._is_hma_required = (
|
||||
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
|
||||
and any(
|
||||
not isinstance(g.kv_cache_spec, FullAttentionSpec)
|
||||
for g in kv_cache_config.kv_cache_groups
|
||||
)
|
||||
)
|
||||
if self._is_hma_required:
|
||||
# TODO(simondanielsson): support non-sliding window hybrids
|
||||
unsupported = [
|
||||
type(g.kv_cache_spec).__name__
|
||||
for g in kv_cache_config.kv_cache_groups
|
||||
if not isinstance(
|
||||
g.kv_cache_spec, (FullAttentionSpec, SlidingWindowSpec)
|
||||
)
|
||||
]
|
||||
if unsupported:
|
||||
raise NotImplementedError(
|
||||
"MoRIIO only supports sliding-window hybrid models (e.g. "
|
||||
"Gemma) under the hybrid KV cache manager. Unsupported KV "
|
||||
f"cache group spec(s): {sorted(set(unsupported))}. Pass "
|
||||
"--disable-hybrid-kv-cache-manager to run without HMA."
|
||||
)
|
||||
if self.mode == MoRIIOMode.WRITE:
|
||||
# TODO(simondanielsson): support HMA in WRITE mode
|
||||
raise NotImplementedError(
|
||||
"MoRIIO WRITE mode does not support hybrid KV cache groups "
|
||||
"(sliding-window attention). Use READ mode, or pass "
|
||||
"--disable-hybrid-kv-cache-manager."
|
||||
)
|
||||
|
||||
sw_sizes_tokens: list[tuple[int, int]] = [
|
||||
(g.kv_cache_spec.sliding_window, g.kv_cache_spec.block_size)
|
||||
if isinstance(g.kv_cache_spec, SlidingWindowSpec)
|
||||
else (0, self.block_size)
|
||||
for g in kv_cache_config.kv_cache_groups
|
||||
]
|
||||
# add 1 to conservatively account for boundary overlap eg window isn't fully
|
||||
# aligned with blocks.
|
||||
self.blocks_per_sw = [
|
||||
cdiv(n_tokens, block_size) + 1 if n_tokens else 0
|
||||
for n_tokens, block_size in sw_sizes_tokens
|
||||
]
|
||||
# In WRITE mode and chunked prefill, we perform the write after the last chunk.
|
||||
# Hence we need to track once we are in the last chunk. We do this by computing
|
||||
# len(block_ids[g]) * block_size[g] for a full attn group g, as hybrid groups
|
||||
# may be clipped and hence not reflect the full sequence length. Here we store
|
||||
# g and block_size[g] for later.
|
||||
self._full_attn_group_idx = 0
|
||||
self._full_attn_block_size = self.block_size
|
||||
for gi, group in enumerate(kv_cache_config.kv_cache_groups):
|
||||
is_full_attn = self.blocks_per_sw[gi] == 0
|
||||
if is_full_attn:
|
||||
self._full_attn_group_idx = gi
|
||||
self._full_attn_block_size = getattr(
|
||||
group.kv_cache_spec, "block_size", self.block_size
|
||||
)
|
||||
break
|
||||
|
||||
self.host_ip = resolve_host_ip(
|
||||
self.kv_transfer_config.kv_connector_extra_config
|
||||
)
|
||||
@@ -409,14 +489,14 @@ class MoRIIOConnectorScheduler:
|
||||
# Requests that need to start recv/send.
|
||||
# New requests are added by update_state_after_alloc in
|
||||
# the scheduler. Used to make metadata passed to Worker.
|
||||
self._reqs_need_recv: dict[ReqId, tuple[Request, list[int]]] = {}
|
||||
self._reqs_need_save: dict[ReqId, tuple[Request, list[int]]] = {}
|
||||
self._reqs_need_recv: dict[ReqId, tuple[Request, BlockIds]] = {}
|
||||
self._reqs_need_save: dict[ReqId, tuple[Request, BlockIds]] = {}
|
||||
# Snapshot of kv_transfer_params for chunked prefill recovery.
|
||||
self._req_kv_params: dict[ReqId, dict] = {}
|
||||
|
||||
# For chunked prefill, we perform layer-wise access within the final chunk.
|
||||
# TODO: Perform transfer at end chunk.
|
||||
self._reqs_need_pending_save: dict[ReqId, tuple[Request, list[int]]] = {}
|
||||
self._reqs_need_pending_save: dict[ReqId, tuple[Request, BlockIds]] = {}
|
||||
|
||||
if self.is_producer:
|
||||
set_role(ROLE.PRODUCER)
|
||||
@@ -442,6 +522,23 @@ class MoRIIOConnectorScheduler:
|
||||
self.transfer_id_to_request_id: dict[TransferId, ReqId] = {}
|
||||
self.request_id_to_transfer_id: dict[ReqId, TransferId] = {}
|
||||
|
||||
def get_exchange_clipped_blocks(
|
||||
self,
|
||||
block_ids: BlockIds,
|
||||
) -> BlockIds:
|
||||
"""Clip the block list based on the layer type.
|
||||
|
||||
Behavior:
|
||||
- sliding window: keep only the last window size blocks
|
||||
- full attention: keep all blocks
|
||||
"""
|
||||
if len(block_ids) == 0 or not self._is_hma_required:
|
||||
return [list(blocks) for blocks in block_ids]
|
||||
return [
|
||||
blocks[-self.blocks_per_sw[i] :] if self.blocks_per_sw[i] > 0 else blocks
|
||||
for i, blocks in enumerate(block_ids)
|
||||
]
|
||||
|
||||
def map_request_id(self, request_id: ReqId, transfer_id: TransferId):
|
||||
self.transfer_id_to_request_id[transfer_id] = request_id
|
||||
self.request_id_to_transfer_id[request_id] = transfer_id
|
||||
@@ -642,7 +739,7 @@ class MoRIIOConnectorScheduler:
|
||||
request_id = request.request_id
|
||||
self.map_request_id(request_id, transfer_id)
|
||||
if params.get("do_remote_decode"):
|
||||
local_block_ids = blocks.get_block_ids()[0]
|
||||
local_block_ids: BlockIds = blocks.get_block_ids()
|
||||
self._reqs_need_save[request.request_id] = (request, local_block_ids)
|
||||
# Snapshot params now so chunked-prefill build_connector_meta
|
||||
# can recover them on the final chunk even if the live
|
||||
@@ -657,17 +754,22 @@ class MoRIIOConnectorScheduler:
|
||||
if "remote_engine_id" in params:
|
||||
if num_external_tokens > 0:
|
||||
# Get unhashed blocks to pull from remote.
|
||||
local_block_ids = blocks.get_block_ids()[0]
|
||||
assert len(local_block_ids) <= len(remote_block_ids)
|
||||
if len(local_block_ids) != len(remote_block_ids):
|
||||
local_block_ids = remote_block_ids[
|
||||
-len(local_block_ids) :
|
||||
]
|
||||
local_block_ids = self.get_exchange_clipped_blocks(
|
||||
blocks.get_block_ids()
|
||||
)
|
||||
# Partial cache hits can lead to fewer local blocks vs
|
||||
# remote blocks, but never more
|
||||
assert all(
|
||||
len(local) <= len(remote)
|
||||
for local, remote in zip(
|
||||
local_block_ids, remote_block_ids, strict=True
|
||||
)
|
||||
)
|
||||
else:
|
||||
# If remote_blocks and num_external_tokens = 0, we have
|
||||
# a full prefix cache hit on the D worker. We need to call
|
||||
# send_notify in _read_blocks to free the memory on the P.
|
||||
local_block_ids = []
|
||||
local_block_ids = [[] for _ in remote_block_ids]
|
||||
|
||||
self._reqs_need_recv[request.request_id] = (
|
||||
request,
|
||||
@@ -818,8 +920,6 @@ class MoRIIOConnectorScheduler:
|
||||
new_block_ids = scheduler_output.scheduled_cached_reqs.new_block_ids[i]
|
||||
|
||||
if new_block_ids is not None:
|
||||
block_ids = new_block_ids[0]
|
||||
# TODO : hybrid attn, etc
|
||||
# A non-disagg request (no kv_transfer_params, e.g. smoke
|
||||
# test) is never registered in _reqs_need_pending_save;
|
||||
# indexing it unconditionally would KeyError and crash the
|
||||
@@ -827,12 +927,16 @@ class MoRIIOConnectorScheduler:
|
||||
if req_id not in self._reqs_need_pending_save:
|
||||
continue
|
||||
req, existing_blocks = self._reqs_need_pending_save[req_id]
|
||||
updated_blocks = list(existing_blocks) + (block_ids)
|
||||
updated_blocks = [
|
||||
existing_blocks[g] + new_block_ids[g]
|
||||
for g in range(len(new_block_ids))
|
||||
]
|
||||
self._reqs_need_pending_save[req_id] = (req, updated_blocks)
|
||||
if (
|
||||
len(self._reqs_need_pending_save[req_id][1]) * self.block_size
|
||||
>= req.num_prompt_tokens
|
||||
):
|
||||
saved_tokens = (
|
||||
len(updated_blocks[self._full_attn_group_idx])
|
||||
* self._full_attn_block_size
|
||||
)
|
||||
if saved_tokens >= req.num_prompt_tokens:
|
||||
# Final chunk: live kv_transfer_params may be cleared,
|
||||
# so prefer the snapshot from update_state_after_alloc.
|
||||
kv_params = self._req_kv_params.pop(
|
||||
@@ -857,7 +961,10 @@ class MoRIIOConnectorScheduler:
|
||||
|
||||
for req_id, (req, block_ids) in self._reqs_need_save.items():
|
||||
kv_params = self._req_kv_params.get(req_id, req.kv_transfer_params or {})
|
||||
if req.num_prompt_tokens > len(block_ids) * self.block_size:
|
||||
tokens_covered = (
|
||||
len(block_ids[self._full_attn_group_idx]) * self._full_attn_block_size
|
||||
)
|
||||
if req.num_prompt_tokens > tokens_covered:
|
||||
# not last chunk prefill
|
||||
self._reqs_need_pending_save[req_id] = (req, block_ids)
|
||||
continue
|
||||
@@ -898,7 +1005,7 @@ class MoRIIOConnectorScheduler:
|
||||
def request_finished(
|
||||
self,
|
||||
request: "Request",
|
||||
block_ids: list[int],
|
||||
block_ids: BlockIds,
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""
|
||||
Once a request is finished, determine whether request blocks
|
||||
@@ -936,7 +1043,8 @@ class MoRIIOConnectorScheduler:
|
||||
if self.mode == MoRIIOMode.WRITE:
|
||||
self._release_write_prefill_blocks(request.request_id, params)
|
||||
else:
|
||||
self._reqs_need_recv[request.request_id] = (request, [])
|
||||
empty_groups: BlockIds = [[] for _ in self.blocks_per_sw]
|
||||
self._reqs_need_recv[request.request_id] = (request, empty_groups)
|
||||
params["do_remote_prefill"] = False
|
||||
return False, None
|
||||
|
||||
@@ -946,10 +1054,9 @@ class MoRIIOConnectorScheduler:
|
||||
):
|
||||
return False, None
|
||||
|
||||
# computed_block_ids = block_ids if all_full else block_ids[:-1]
|
||||
computed_block_ids = block_ids
|
||||
# If prompt < block_size, no xfer so free blocks immediately.
|
||||
delay_free_blocks = len(computed_block_ids) > 0
|
||||
# On producer
|
||||
computed_block_ids = self.get_exchange_clipped_blocks(block_ids)
|
||||
delay_free_blocks = any(len(group) > 0 for group in computed_block_ids)
|
||||
|
||||
if delay_free_blocks:
|
||||
# Prefill request on remote. It will be read from D upon completion
|
||||
@@ -1102,6 +1209,12 @@ class MoRIIOConnectorWorker:
|
||||
self.kv_transfer_config = vllm_config.kv_transfer_config
|
||||
self.is_producer = self.kv_transfer_config.is_kv_producer
|
||||
self.layer_to_spec = build_layer_to_spec(kv_cache_config)
|
||||
# we transfer by layer, but store by kv cache group
|
||||
self.layer_to_group: dict[str, int] = {
|
||||
layer_name: group_idx
|
||||
for group_idx, group in enumerate(kv_cache_config.kv_cache_groups)
|
||||
for layer_name in group.layer_names
|
||||
}
|
||||
|
||||
if self.is_producer:
|
||||
set_role(ROLE.PRODUCER)
|
||||
@@ -1789,8 +1902,6 @@ class MoRIIOConnectorWorker:
|
||||
self.block_size,
|
||||
)
|
||||
self.block_size = first_geometry.block_size
|
||||
# TODO(tms): self.block_len needs to be per-layer for sliding window,
|
||||
# hybrid attn, etc
|
||||
# block size in bytes
|
||||
self.block_len = first_geometry.block_len
|
||||
self.kv_cache_shape = first_kv_cache.shape
|
||||
@@ -1803,14 +1914,30 @@ class MoRIIOConnectorWorker:
|
||||
self.layer_base_addr_index = {}
|
||||
base_addr_idx = 0
|
||||
|
||||
hma_enabled = (
|
||||
not self.vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
|
||||
)
|
||||
for layer_name in kv_caches:
|
||||
self.layer_base_addr_index[layer_name] = base_addr_idx
|
||||
geometry = self._get_layer_transfer_geometry(layer_name)
|
||||
if geometry.block_size != self.block_size:
|
||||
if hma_enabled:
|
||||
if geometry.block_len != self.block_len:
|
||||
raise ValueError(
|
||||
"MoRIIO KV cache block length mismatch for layer "
|
||||
f"{layer_name}: {geometry.block_len} != {self.block_len}"
|
||||
)
|
||||
elif geometry.block_size != self.block_size:
|
||||
raise ValueError(
|
||||
"MoRIIO KV cache block size mismatch for layer "
|
||||
f"{layer_name}: {geometry.block_size} != {self.block_size}"
|
||||
)
|
||||
# num_blocks is advertised as a single scalar to the peer, so must it
|
||||
# be uniform
|
||||
if geometry.num_blocks != self.num_blocks:
|
||||
raise ValueError(
|
||||
"MoRIIO KV cache num_blocks mismatch for layer "
|
||||
f"{layer_name}: {geometry.num_blocks} != {self.num_blocks}"
|
||||
)
|
||||
self.block_lens[layer_name] = geometry.block_len
|
||||
for cache, region_len in self._iter_layer_registration_regions(layer_name):
|
||||
base_addr = cache.data_ptr()
|
||||
@@ -2455,12 +2582,16 @@ class MoRIIOConnectorWorker:
|
||||
self.remote_dp_size_local = int(meta.remote_dp_size_local)
|
||||
else:
|
||||
self.remote_dp_size_local = int(meta.remote_dp_size)
|
||||
# WRITE does not support HMA, so unwrap blocks into flat lists.
|
||||
# remote_block_ids can itself be empty so need to be careful when unwrapping.
|
||||
local_block_ids = meta.local_block_ids[0]
|
||||
remote_block_ids = meta.remote_block_ids[0] if meta.remote_block_ids else []
|
||||
self.schedule_write_blocks(
|
||||
request_id=req_id,
|
||||
transfer_id=meta.transfer_id,
|
||||
dst_engine_id=meta.remote_engine_id,
|
||||
local_block_ids=meta.local_block_ids,
|
||||
remote_block_ids=meta.remote_block_ids,
|
||||
local_block_ids=local_block_ids,
|
||||
remote_block_ids=remote_block_ids,
|
||||
layer_name=layer_name,
|
||||
kv_layer=kv_layer,
|
||||
remote_notify_port=meta.remote_notify_port,
|
||||
@@ -2607,8 +2738,8 @@ class MoRIIOConnectorWorker:
|
||||
|
||||
def _read_blocks(
|
||||
self,
|
||||
local_block_ids: list[int],
|
||||
remote_block_ids: list[int],
|
||||
local_block_ids: BlockIds,
|
||||
remote_block_ids: BlockIds,
|
||||
dst_engine_id: str,
|
||||
request_id: str,
|
||||
transfer_id: str,
|
||||
@@ -2653,10 +2784,11 @@ class MoRIIOConnectorWorker:
|
||||
sess_idx = list(self.layer_name_to_local_kv_cache_metadata.keys()).index(
|
||||
layer_name
|
||||
)
|
||||
group_idx = self.layer_to_group[layer_name]
|
||||
offs = self._compute_block_transfer_offsets(
|
||||
layer_name,
|
||||
local_block_ids,
|
||||
remote_block_ids,
|
||||
local_block_ids[group_idx],
|
||||
remote_block_ids[group_idx],
|
||||
remote_moriio_meta,
|
||||
remote_tp_size=remote_tp_size,
|
||||
remote_engine_id=remote_dp_engine_id,
|
||||
|
||||
Reference in New Issue
Block a user