[HiSparse] Own GPU residency instead of editing core block-table rows (#60647)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Matthew Bonanni
2026-10-09 21:25:44 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 9bd074a996
commit d5f356dae0
15 changed files with 429 additions and 128 deletions
@@ -36,7 +36,11 @@ from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.hisparse.coordinator import get_hisparse_coordinator
from vllm.v1.hisparse.layout import get_hisparse_steady_state_concurrency
from vllm.v1.hisparse.types import SparseKVOffloadCommand, SparseKVRowMirror
from vllm.v1.hisparse.types import (
SparseKVOffloadCommand,
SparseKVResidencyUpdate,
SparseKVRowMirror,
)
from vllm.v1.outputs import KVConnectorOutput
if TYPE_CHECKING:
@@ -56,6 +60,7 @@ class HiSparseConnectorMetadata(KVConnectorMetadata):
source_block_ids: tuple[int, ...]
row_mirrors: dict[str, tuple[SparseKVRowMirror, ...]]
all_context_pages_resident: bool
residency_updates: dict[str, SparseKVResidencyUpdate]
@dataclass
@@ -143,7 +148,7 @@ class HiSparseConnectorScheduler:
scheduler_output.num_scheduled_tokens.items()
)
)
# Runs first: it plans this step's transfers and table updates.
# Runs first: it plans this step's transfers and residency changes.
self.coordinator.advance_scheduled(
(
request_id,
@@ -151,9 +156,6 @@ class HiSparseConnectorScheduler:
)
for request_id, start, count in scheduled_requests
)
scheduler_output.block_table_updates = (
self.coordinator.take_block_table_updates() or None
)
command = self.coordinator.build_offload_command()
host_block_copies = self.coordinator.take_host_block_copies()
source_group_id = self.coordinator.host_group_id
@@ -192,6 +194,9 @@ class HiSparseConnectorScheduler:
tuple(source_block_ids),
row_mirrors,
self.coordinator.all_context_pages_resident(scheduled_requests),
self.coordinator.take_residency_updates(
scheduler_output.num_scheduled_tokens
),
)
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
@@ -26,7 +26,11 @@ from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.torch_utils import current_stream
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
from vllm.v1.hisparse.layout import HISPARSE_HOT_SUFFIX
from vllm.v1.hisparse.runtime import HiSparseCacheHandle, release_pinned_state
from vllm.v1.hisparse.runtime import (
HiSparseCacheHandle,
release_pinned_state,
update_hisparse_residency,
)
from vllm.v1.hisparse.types import SparseKVPageTransfer, SparseKVRowMirror
from vllm.v1.kv_cache_interface import (
HiSparseHotSpec,
@@ -334,6 +338,10 @@ class HiSparseConnectorWorker:
"HiSparse request-state mapping does not match max_num_seqs."
)
self.hot_backing = hot_backing
self.residency: torch.Tensor | None = next(
(cache.residency for cache in cache_handles if cache.residency is not None),
None,
)
self._pending_invalid_block_ids: list[int] = []
# Destination block ids of host copies this worker has run.
self._completed_host_copy_dst_ids: list[int] = []
@@ -437,6 +445,15 @@ class HiSparseConnectorWorker:
self._pending_invalid_block_ids.extend(metadata.source_block_ids)
if request_state_indices is not None:
self.set_request_state_indices(request_state_indices)
if metadata.residency_updates:
assert self.residency is not None
assert request_state_indices is not None and request_ids is not None
update_hisparse_residency(
self.residency,
metadata.residency_updates,
request_ids,
request_state_indices,
)
def _clear_forward_mirror_state(self) -> None:
self._per_layer_mirrored.clear()
@@ -243,7 +243,7 @@ class HiSparseMLAIndexGroup(SparseMLAIndexGroup):
layer_index,
logical_topk_indices,
req_id_per_token,
leader.block_table,
leader.batch_block_table(),
leader.view.block_size,
block_stride_rows=leader.view.attention_block_stride,
return_valid_counts=return_valid_counts,
@@ -268,8 +268,11 @@ class HiSparseMLAIndexGroup(SparseMLAIndexGroup):
assert staging_plan is not None
resident_cache = None
if cache.view is not None and cache.block_table is not None:
state_indices = cache.runtime.request_state_indices
assert state_indices is not None
staging_plan.ensure_gpu_sources(
cache.block_table[attn_metadata.num_decodes :],
cache.block_table,
state_indices[attn_metadata.num_decodes :],
cache.view.block_size,
)
resident_cache = cache.view.cache
@@ -291,8 +294,11 @@ class HiSparseMLAIndexGroup(SparseMLAIndexGroup):
plan = prefill.host_staging_plan if prefill is not None else None
assert plan is not None
assert cache.view is not None and cache.block_table is not None
state_indices = cache.runtime.request_state_indices
assert state_indices is not None
plan.ensure_gpu_sources(
cache.block_table[attn_metadata.num_decodes :],
cache.block_table,
state_indices[attn_metadata.num_decodes :],
cache.view.block_size,
)
assert plan.gpu_row_ids is not None
-3
View File
@@ -304,9 +304,6 @@ class SchedulerOutput:
# CoW copies to apply after zeroing new blocks and before forward.
kv_cache_block_copies: list[KVCacheBlockCopy] | None = None
# Complete block-table rows that replace incrementally appended block IDs.
block_table_updates: dict[str, tuple[list[int], ...]] | None = None
# Scheduler-local; always None by the time this reaches a worker.
kv_connector_block_state: KVConnectorBlockState | None = None
+38 -4
View File
@@ -2503,6 +2503,10 @@ class HiSparseResidentManager(_HiSparseAuxiliaryManager):
max_admission_blocks_per_request: int
def __init__(self, kv_cache_spec: HiSparseResidentSpec, **kwargs) -> None:
super().__init__(kv_cache_spec, **kwargs)
self.residency_changes: dict[str, dict[int, KVCacheBlock]] = {}
def get_num_blocks_to_allocate(
self,
request_id: str,
@@ -2546,7 +2550,7 @@ class HiSparseResidentManager(_HiSparseAuxiliaryManager):
tail_page = (num_tokens - 1) // self.block_size
if len(blocks) <= tail_page:
blocks.extend([self._null_block] * (tail_page + 1 - len(blocks)))
if blocks[tail_page].is_null:
if self.get_resident_page(request_id, tail_page) is None:
blocks[tail_page] = self.block_pool.get_new_blocks(1)[0]
def add_local_computed_blocks(
@@ -2580,25 +2584,55 @@ class HiSparseResidentManager(_HiSparseAuxiliaryManager):
def pop_blocks_for_free(self, request_id: str) -> list[KVCacheBlock]:
assert self.coordinator is not None
self.coordinator.free(request_id)
return super().pop_blocks_for_free(request_id)
blocks = self.get_residency_row(request_id)
self.residency_changes.pop(request_id, None)
super().pop_blocks_for_free(request_id)
return blocks
def adopt_resident_page(
self, request_id: str, block_idx: int, block: KVCacheBlock
) -> bool:
"""Point a null prefix page at a pinned GPU copy of its contents."""
blocks = self.req_to_blocks.get(request_id)
if blocks is None or block_idx >= len(blocks) or not blocks[block_idx].is_null:
if blocks is None or block_idx >= len(blocks):
return False
blocks[block_idx] = block
if self.get_resident_page(request_id, block_idx) is not None:
return False
self.residency_changes.setdefault(request_id, {})[block_idx] = block
return True
def drop_resident_page(
self, request_id: str, block_idx: int
) -> KVCacheBlock | None:
"""Stop reading a page from the GPU; return the block it was read from."""
block = self.get_resident_page(request_id, block_idx)
if block is not None:
changes = self.residency_changes.setdefault(request_id, {})
changes[block_idx] = self._null_block
return block
def get_resident_page(self, request_id: str, block_idx: int) -> KVCacheBlock | None:
blocks = self.req_to_blocks.get(request_id)
if blocks is None or block_idx >= len(blocks):
return None
block = blocks[block_idx]
changes = self.residency_changes.get(request_id)
if changes:
block = changes.get(block_idx, block)
return None if block.is_null else block
def get_residency_row(self, request_id: str, start: int = 0) -> list[KVCacheBlock]:
"""The request's row from ``start`` with residency changes applied. Unlike
``get_resident_page``, host-only pages stay as the null block."""
blocks = self.req_to_blocks.get(request_id, [])[start:]
changes = self.residency_changes.get(request_id)
if not changes:
return blocks
return [
changes.get(block_idx, block)
for block_idx, block in enumerate(blocks, start)
]
def get_manager_for_kv_cache_spec(
kv_cache_spec: KVCacheSpec,
+22 -6
View File
@@ -214,10 +214,26 @@ def bind_hisparse_kv_caches(
for tensor_config in kv_cache_config.kv_cache_tensors
for layer_index, name in enumerate(tensor_config.layers)
}
resident_source_index = 0
for group_id, group in enumerate(kv_cache_config.kv_cache_groups):
if not isinstance(group.kv_cache_spec, HiSparseResidentSpec):
continue
resident_groups = [
(group_id, group)
for group_id, group in enumerate(kv_cache_config.kv_cache_groups)
if isinstance(group.kv_cache_spec, HiSparseResidentSpec)
]
residency: torch.Tensor | None = None
if resident_groups:
table_shapes = {
block_tables.input_block_tables[group_id].shape
for group_id, _ in resident_groups
}
assert len(table_shapes) == 1
max_num_reqs, max_num_pages = table_shapes.pop()
residency = torch.zeros(
(max_num_reqs, len(resident_groups), max_num_pages),
dtype=torch.int32,
device=block_tables.input_block_tables[resident_groups[0][0]].device,
)
for resident_source_index, (group_id, group) in enumerate(resident_groups):
assert residency is not None
for cache_name in group.layer_names:
assert cache_name.endswith(HISPARSE_RESIDENT_SUFFIX)
layer_name = cache_name[: -len(HISPARSE_RESIDENT_SUFFIX)]
@@ -230,13 +246,13 @@ def bind_hisparse_kv_caches(
block_stride=tensor_config.block_stride,
num_blocks=kv_cache_config.num_blocks,
block_size=group.kv_cache_spec.block_size,
block_table=block_tables.input_block_tables[group_id],
block_table=residency[:, resident_source_index],
slot_mapping=block_tables.slot_mappings[group_id],
)
cache_handle.residency = residency
assert cache_handle.view is not None
kv_caches[cache_name] = cache_handle.view.cache
cache_handle.runtime.resident_source_index = resident_source_index
resident_source_index += 1
hot_backing: torch.Tensor | None = None
cache_handles: list[HiSparseCacheHandle] = []
+70 -41
View File
@@ -22,6 +22,7 @@ from vllm.v1.hisparse.types import (
ACTIVE_TAIL_PAGES,
SparseKVOffloadCommand,
SparseKVPageTransfer,
SparseKVResidencyUpdate,
SparseKVRowMirror,
)
from vllm.v1.kv_cache_interface import (
@@ -65,6 +66,10 @@ class _HiSparseRequestState:
unpinned_pages: set[int] = field(default_factory=set)
# Prefix pages whose GPU copies are adopted after the admitting allocation.
pages_to_adopt: int = 0
# The worker holds this request's residency for every page below
# synced_pages, except the stale pages changed since it was sent.
synced_pages: int = 0
stale_pages: set[int] = field(default_factory=set)
@dataclass
@@ -86,9 +91,10 @@ class HiSparseCoordinator:
GPU-resident pages are a write-back cache of the host tier. Once a page's
host copy is durable and its owner can read from host (it has, or has
asked for, a hot region), the page's allocation reference is released with
``BlockPool.unpin_blocks``: the block stays in the block table and keeps
being read, but the pool counts it as free and may hand it out, at which
point the owner's page is nulled and its block table republished.
``BlockPool.unpin_blocks``: the block keeps being read, but the pool counts
it as free and may hand it out, at which point the owner stops reading it.
Residency changes reach the worker through ``take_residency_updates``, not
the block table, which stays append-only.
"""
def __init__(
@@ -167,7 +173,6 @@ class HiSparseCoordinator:
self._retained_copies: dict[int, tuple[KVCacheBlock, KVCacheBlock]] = {}
# request -> prefix length an external load is still filling in.
self._pending_imports: dict[str, int] = {}
self.block_table_updates: set[str] = set()
# Requests whose resident pages were all non-null when last scanned.
# Pages only become null at admission or import (before the first
# scan) and when the pool reuses an unpinned page (_lose_page).
@@ -269,7 +274,8 @@ class HiSparseCoordinator:
if manager.adopt_resident_page(request_id, host_idx, block):
manager.block_pool.touch([block])
state.pinned_clean.add(host_idx)
self.block_table_updates.add(request_id)
if host_idx < state.synced_pages:
state.stale_pages.add(host_idx)
def _record_copies(self, request_id: str, num_computed_tokens: int) -> None:
"""Index the GPU copies of just-published host blocks for later hits."""
@@ -342,14 +348,11 @@ class HiSparseCoordinator:
) -> list[KVCacheBlock] | None:
blocks: list[KVCacheBlock] = []
for manager in self.resident_managers:
req_blocks = manager.req_to_blocks.get(request_id)
if (
req_blocks is None
or page_idx >= len(req_blocks) - ACTIVE_TAIL_PAGES
or req_blocks[page_idx].is_null
):
num_pages = len(manager.req_to_blocks.get(request_id, ()))
block = manager.get_resident_page(request_id, page_idx)
if page_idx >= num_pages - ACTIVE_TAIL_PAGES or block is None:
return None
blocks.append(req_blocks[page_idx])
blocks.append(block)
return blocks
def _unpin_page(
@@ -392,16 +395,15 @@ class HiSparseCoordinator:
if state is None:
return
for manager in self.resident_managers:
blocks = manager.req_to_blocks.get(request_id)
if blocks is None or page_idx >= len(blocks) or blocks[page_idx].is_null:
block = manager.drop_resident_page(request_id, page_idx)
if block is None:
continue
block = blocks[page_idx]
blocks[page_idx] = manager._null_block
owners = self._owners.get(block.block_id)
if owners is not None:
owners.discard((request_id, page_idx))
state.unpinned_pages.discard(page_idx)
self.block_table_updates.add(request_id)
if page_idx < state.synced_pages:
state.stale_pages.add(page_idx)
for hot_manager in self.hot_managers:
hot_manager.require_hot(request_id)
@@ -627,11 +629,10 @@ class HiSparseCoordinator:
mirrors: list[SparseKVRowMirror] = []
for request_id, num_computed_tokens, num_scheduled_tokens in requests:
host_blocks = self.host_manager.req_to_blocks.get(request_id)
resident_blocks = [
manager.req_to_blocks.get(request_id)
if host_blocks is None or any(
request_id not in manager.req_to_blocks
for manager in self.resident_managers
]
if host_blocks is None or any(blocks is None for blocks in resident_blocks):
):
continue
token_position = num_computed_tokens
end_position = token_position + num_scheduled_tokens
@@ -644,14 +645,12 @@ class HiSparseCoordinator:
# window only ever moves forward, so dropping them here would
# leave their host rows permanently stale.
source_starts = []
for blocks in resident_blocks:
assert blocks is not None
if page_idx >= len(blocks) or blocks[page_idx].is_null:
for manager in self.resident_managers:
block = manager.get_resident_page(request_id, page_idx)
if block is None:
break
source_starts.append(
blocks[page_idx].block_id * block_size + row_offset
)
if len(source_starts) != len(resident_blocks):
source_starts.append(block.block_id * block_size + row_offset)
if len(source_starts) != len(self.resident_managers):
token_position += num_rows
continue
host_block_idx = page_idx
@@ -693,21 +692,53 @@ class HiSparseCoordinator:
if any(
block.is_null
for manager in self.resident_managers
for block in manager.req_to_blocks.get(request_id, ())
for block in manager.get_residency_row(request_id)
):
return False
self._resident_requests.add(request_id)
return True
def take_block_table_updates(self) -> dict[str, tuple[list[int], ...]]:
updates = {
request_id: tuple(
[block.block_id for block in manager.req_to_blocks.get(request_id, [])]
for manager in self.managers
def take_residency_updates(
self, request_ids: Iterable[str]
) -> dict[str, SparseKVResidencyUpdate]:
"""Resident block ids the worker lacks for the scheduled requests:
stale pages plus pages appended since the last update.
Requests that are not scheduled keep their changes until they are:
only a scheduled request's residency is read.
"""
updates: dict[str, SparseKVResidencyUpdate] = {}
if not self.resident_managers:
return updates
first = self.resident_managers[0]
for request_id in request_ids:
state = self._get_request_state(request_id)
num_pages = len(first.req_to_blocks.get(request_id, ()))
start = state.synced_pages
if not state.stale_pages and start >= num_pages:
continue
stale_pages = sorted(state.stale_pages)
block_ids: list[list[int]] = []
for manager in self.resident_managers:
null_block_id = manager.block_pool.null_block.block_id
group_block_ids = [
null_block_id if block is None else block.block_id
for block in (
manager.get_resident_page(request_id, page_idx)
for page_idx in stale_pages
)
]
group_block_ids.extend(
block.block_id
for block in manager.get_residency_row(request_id, start)
)
block_ids.append(group_block_ids)
updates[request_id] = SparseKVResidencyUpdate(
pages=[*stale_pages, *range(start, num_pages)],
block_ids=tuple(block_ids),
)
for request_id in self.block_table_updates
}
self.block_table_updates.clear()
state.stale_pages.clear()
state.synced_pages = num_pages
return updates
def build_offload_command(self) -> SparseKVOffloadCommand | None:
@@ -846,9 +877,7 @@ class HiSparseCoordinator:
# planned, so its copies do not prove that the KV was computed.
state.publication = None
for manager in self.resident_managers:
blocks = manager.req_to_blocks.get(request_id)
if not blocks:
continue
blocks = manager.get_residency_row(request_id)
# Tail first, so a prefix loses its tail pages before its head.
for page_idx in reversed(range(len(blocks))):
block = blocks[page_idx]
@@ -865,7 +894,7 @@ class HiSparseCoordinator:
manager.block_pool.unpin_blocks([block], self._on_block_reused)
else:
continue
blocks[page_idx] = manager._null_block
manager.drop_resident_page(request_id, page_idx)
def get_hisparse_coordinator(
+68 -8
View File
@@ -7,10 +7,11 @@ from __future__ import annotations
import math
import mmap
import time
from collections.abc import Callable
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, TypeAlias
import numpy as np
import psutil
import torch
@@ -21,8 +22,9 @@ from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.math_utils import cdiv, round_up
from vllm.utils.mem_utils import get_max_shared_memory_bytes
from vllm.utils.torch_utils import current_stream
from vllm.utils.torch_utils import async_tensor_h2d, current_stream
from vllm.v1.attention.backend import max_decode_query_len
from vllm.v1.hisparse.types import SparseKVResidencyUpdate
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
from vllm.v1.simple_kv_offload.cuda_mem_ops import pin_tensor
@@ -499,16 +501,19 @@ class HiSparsePrefillStagingPlan:
def ensure_gpu_sources(
self,
resident_block_table: torch.Tensor,
resident_state_rows: torch.Tensor,
state_indices: torch.Tensor,
resident_block_size: int,
) -> None:
"""Resolve which staged rows can be served from the resident cache.
Computed once per plan (the resident block table is shared by every
layer in the group); non-null resident pages become miss_mask=0 rows
gathered device-to-device by ``gather_prefill_cache``.
``resident_state_rows`` is a resident group's persistent table by
request state row and ``state_indices`` holds each staged request's
state row. Computed once per plan and resident group (every layer in
the group shares the table); non-null resident pages become
miss_mask=0 rows gathered device-to-device by ``gather_prefill_cache``.
"""
source_key = (resident_block_table.data_ptr(), resident_block_size)
source_key = (resident_state_rows.data_ptr(), resident_block_size)
if self.gpu_source_key == source_key:
return
block_size = self.block_size
@@ -519,8 +524,11 @@ class HiSparsePrefillStagingPlan:
host_ids = self.row_ids[0].view(num_unique, block_size)[:, 0] // block_size
new_bt = self.block_table.to(torch.int64)
num_rows, num_cols = new_bt.shape
if num_rows == 0 or resident_block_table.shape[0] < num_rows:
if num_rows == 0 or state_indices.shape[0] < num_rows:
return
resident_block_table = resident_state_rows.index_select(
0, state_indices[:num_rows].clamp(min=0)
)
# One representative (row, col) per unique host block: any request
# referencing the block holds an equivalent (refcounted) resident view.
flat_pos = torch.arange(num_rows * num_cols, device=device)
@@ -1107,11 +1115,50 @@ class HiSparseRuntime:
return physical_topk_indices
def update_hisparse_residency(
residency: torch.Tensor,
updates: Mapping[str, SparseKVResidencyUpdate],
request_ids: Sequence[str],
request_state_indices: torch.Tensor,
) -> None:
"""Apply scheduled requests' residency changes.
``residency`` is ``[state rows, resident groups, pages]``: the GPU block of
each page, or block 0 when the page is read from the host.
"""
if not updates:
return
batch_rows = {request_id: row for row, request_id in enumerate(request_ids)}
num_groups = residency.shape[1]
rows: list[int] = []
pages: list[int] = []
block_ids: list[list[int]] = [[] for _ in range(num_groups)]
for request_id, update in updates.items():
rows.append(batch_rows[request_id])
pages.extend(update.pages)
for group_block_ids, update_block_ids in zip(
block_ids, update.block_ids, strict=True
):
group_block_ids.extend(update_block_ids)
host_values = np.empty((len(pages), 2 + num_groups), dtype=np.int32)
host_values[:, 0] = np.repeat(
rows, [len(update.pages) for update in updates.values()]
)
host_values[:, 1] = pages
host_values[:, 2:] = np.asarray(block_ids, dtype=np.int32).T
values = async_tensor_h2d(
host_values, device=request_state_indices.device, dtype=torch.int32
)
state_rows = request_state_indices[values[:, 0]]
residency[state_rows, :, values[:, 1]] = values[:, 2:]
class HiSparseCacheHandle:
"""Attention-facing handle for resident KV and sparse offload state."""
def __init__(self, runtime: HiSparseRuntime) -> None:
self.view: PagedCacheView | None = None
self.residency: torch.Tensor | None = None
self.block_table: torch.Tensor | None = None
self.source_block_table: torch.Tensor | None = None
self.slot_mapping: torch.Tensor | None = None
@@ -1128,6 +1175,7 @@ class HiSparseCacheHandle:
# Speculator layers write their rows after the target forward.
self.draft_layer = False
self.index_group_caches: list[HiSparseCacheHandle] = [self]
self._batch_block_table: torch.Tensor | None = None
def prepare_group_for_batch(self, attn_metadata: Any | None) -> None:
assert self.runtime.is_group_leader
@@ -1136,6 +1184,7 @@ class HiSparseCacheHandle:
def _prepare_for_batch(self, attn_metadata: Any | None) -> None:
self.dummy_batch = attn_metadata is None
self._batch_block_table = None
self.runtime.begin_forward()
self.num_actual_tokens = (
attn_metadata.num_actual_tokens if attn_metadata is not None else 0
@@ -1151,6 +1200,17 @@ class HiSparseCacheHandle:
not self.decode_batch or self.runtime.eager_host_mirror
)
def batch_block_table(self) -> torch.Tensor:
"""Resident rows by this step's batch row, for the prefill paths."""
if self._batch_block_table is None:
assert self.block_table is not None
indices = self.runtime.request_state_indices
assert indices is not None
self._batch_block_table = self.block_table.index_select(
0, indices.clamp(min=0)
)
return self._batch_block_table
def write_target(
self, num_input_rows: int, num_slot_rows: int
) -> tuple[torch.Tensor, torch.Tensor, int]:
+9
View File
@@ -28,6 +28,15 @@ class SparseKVRowMirror:
num_rows: int
@dataclass(frozen=True)
class SparseKVResidencyUpdate:
"""GPU block ids of some of a request's resident pages, per resident group.
A null block id means the page is read from the host."""
pages: list[int]
block_ids: tuple[list[int], ...]
@dataclass
class SparseKVOffloadCommand:
"""Opaque scheduler-to-worker command for one model step."""
+1 -5
View File
@@ -1224,17 +1224,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
def update_requests(self, scheduler_output: SchedulerOutput) -> None:
# Add new blocks and update num_computed_tokens for the existing requests.
reqs = scheduler_output.scheduled_cached_reqs
table_updates = scheduler_output.block_table_updates or {}
for req_id, block_ids in table_updates.items():
if (idx := self.req_states.req_id_to_index.get(req_id)) is not None:
self.block_tables.append_block_ids(idx, block_ids, overwrite=True)
num_computed_tokens_np = self.req_states.num_computed_tokens_np
for req_id, num_computed_tokens, req_new_block_ids in zip(
reqs.req_ids, reqs.num_computed_tokens, reqs.new_block_ids
):
req_index = self.req_states.req_id_to_index[req_id]
num_computed_tokens_np[req_index] = num_computed_tokens
if req_new_block_ids is not None and req_id not in table_updates:
if req_new_block_ids is not None:
self.block_tables.append_block_ids(
req_index, req_new_block_ids, overwrite=False
)