[v1] Support multiple KV cache groups in GPU model runner (#17945)
Signed-off-by: Chen Zhang <zhangch99@outlook.com>
This commit is contained in:
@@ -32,9 +32,16 @@ class KVCacheBlocks:
|
||||
"""Creates a new KVCacheBlocks instance with no blocks."""
|
||||
return cls([])
|
||||
|
||||
def get_block_ids(self) -> list[int]:
|
||||
"""Converts the KVCacheBlocks instance to a list of block IDs."""
|
||||
return [block.block_id for block in self.blocks]
|
||||
def get_block_ids(self) -> list[list[int]]:
|
||||
"""
|
||||
Converts the KVCacheBlocks instance to block_ids.
|
||||
|
||||
Returns:
|
||||
list[list[int]]: A two-level list where
|
||||
* the outer list corresponds to KV cache groups (only 1 group now)
|
||||
* each inner list contains the block_ids of the blocks in that group
|
||||
"""
|
||||
return [[block.block_id for block in self.blocks]]
|
||||
|
||||
def get_unhashed_block_ids(self) -> list[int]:
|
||||
"""Get block_ids of unhashed blocks from KVCacheBlocks instance."""
|
||||
@@ -300,9 +307,9 @@ class KVCacheManager:
|
||||
self,
|
||||
request: Request,
|
||||
num_running_requests: int,
|
||||
) -> int:
|
||||
) -> list[int]:
|
||||
"""Calculate the number of common prefix blocks shared by all requests
|
||||
in the RUNNING state.
|
||||
in the RUNNING state for each kv cache group.
|
||||
|
||||
The function determines this by selecting any request and iterating
|
||||
through its blocks. A block is considered a common prefix block if its
|
||||
@@ -332,11 +339,14 @@ class KVCacheManager:
|
||||
requests in the current step.
|
||||
|
||||
Returns:
|
||||
int: The number of common prefix blocks.
|
||||
list[int]: The number of common prefix blocks for each kv cache
|
||||
group.
|
||||
"""
|
||||
assert request.status == RequestStatus.RUNNING
|
||||
return self.single_type_manager.get_num_common_prefix_blocks(
|
||||
request.request_id, num_running_requests)
|
||||
return [
|
||||
self.single_type_manager.get_num_common_prefix_blocks(
|
||||
request.request_id, num_running_requests)
|
||||
]
|
||||
|
||||
def free_block_hashes(self, request: Request) -> None:
|
||||
"""Discard the block hashes for the request.
|
||||
@@ -354,10 +364,8 @@ class KVCacheManager:
|
||||
"""
|
||||
return self.block_pool.take_events()
|
||||
|
||||
def get_block_ids(self, request_id: str) -> list[int]:
|
||||
def get_block_ids(self, request_id: str) -> list[list[int]]:
|
||||
"""Get the block ids of a request."""
|
||||
assert request_id in self.single_type_manager.req_to_blocks
|
||||
return [
|
||||
block.block_id
|
||||
for block in self.single_type_manager.req_to_blocks[request_id]
|
||||
]
|
||||
return KVCacheBlocks(self.single_type_manager.req_to_blocks[request_id]
|
||||
).get_block_ids()
|
||||
|
||||
@@ -577,14 +577,12 @@ def create_kv_cache_group_specs(
|
||||
"""
|
||||
kv_cache_groups = []
|
||||
for layer_names_one_group in grouped_layer_names:
|
||||
layer_spec = kv_cache_spec[layer_names_one_group[0]]
|
||||
assert all(
|
||||
kv_cache_spec[layer_name] == layer_spec
|
||||
for layer_name in layer_names_one_group[1:]), (
|
||||
"All layers in the same KV cache group must share the same "
|
||||
"KVCacheSpec.")
|
||||
layer_specs = [
|
||||
kv_cache_spec[layer_name] for layer_name in layer_names_one_group
|
||||
]
|
||||
merged_layer_spec = layer_specs[0].merge(layer_specs)
|
||||
kv_cache_groups.append(
|
||||
KVCacheGroupSpec(layer_names_one_group, layer_spec))
|
||||
KVCacheGroupSpec(layer_names_one_group, merged_layer_spec))
|
||||
return kv_cache_groups
|
||||
|
||||
|
||||
@@ -683,6 +681,7 @@ def unify_hybrid_kv_cache_specs(kv_cache_spec: dict[str, KVCacheSpec]):
|
||||
head_size=spec.head_size,
|
||||
dtype=spec.dtype,
|
||||
use_mla=spec.use_mla,
|
||||
sliding_window=spec.sliding_window,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ class NewRequestData:
|
||||
mm_hashes: list[str]
|
||||
mm_positions: list[PlaceholderRange]
|
||||
sampling_params: SamplingParams
|
||||
block_ids: list[int]
|
||||
block_ids: list[list[int]]
|
||||
num_computed_tokens: int
|
||||
lora_request: Optional[LoRARequest]
|
||||
|
||||
@@ -34,7 +34,7 @@ class NewRequestData:
|
||||
def from_request(
|
||||
cls,
|
||||
request: Request,
|
||||
block_ids: list[int],
|
||||
block_ids: list[list[int]],
|
||||
) -> NewRequestData:
|
||||
return cls(
|
||||
req_id=request.request_id,
|
||||
@@ -85,7 +85,7 @@ class CachedRequestData:
|
||||
# request's block IDs instead of appending to the existing block IDs.
|
||||
resumed_from_preemption: bool
|
||||
new_token_ids: list[int]
|
||||
new_block_ids: list[int]
|
||||
new_block_ids: list[list[int]]
|
||||
num_computed_tokens: int
|
||||
|
||||
@classmethod
|
||||
@@ -94,7 +94,7 @@ class CachedRequestData:
|
||||
request: Request,
|
||||
resumed_from_preemption: bool,
|
||||
new_token_ids: list[int],
|
||||
new_block_ids: list[int],
|
||||
new_block_ids: list[list[int]],
|
||||
) -> CachedRequestData:
|
||||
return cls(
|
||||
req_id=request.request_id,
|
||||
@@ -131,9 +131,9 @@ class SchedulerOutput:
|
||||
# E.g., if a request has [0, 1], it could mean the vision encoder needs
|
||||
# to process that the request's 0-th and 1-th images in the current step.
|
||||
scheduled_encoder_inputs: dict[str, list[int]]
|
||||
# Number of common prefix blocks for all requests.
|
||||
# Number of common prefix blocks for all requests in each KV cache group.
|
||||
# This can be used for cascade attention.
|
||||
num_common_prefix_blocks: int
|
||||
num_common_prefix_blocks: list[int]
|
||||
|
||||
# Request IDs that are finished in between the previous and the current
|
||||
# steps. This is used to notify the workers about the finished requests
|
||||
|
||||
@@ -173,7 +173,7 @@ class Scheduler(SchedulerInterface):
|
||||
# uses structured decoding.
|
||||
structured_output_request_ids: dict[str, int] = {}
|
||||
|
||||
req_to_new_block_ids: dict[str, list[int]] = {}
|
||||
req_to_new_block_ids: dict[str, list[list[int]]] = {}
|
||||
num_scheduled_tokens: dict[str, int] = {}
|
||||
token_budget = self.max_num_scheduled_tokens
|
||||
# Encoder-related.
|
||||
@@ -477,7 +477,8 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
# Get the longest common prefix among all requests in the running queue.
|
||||
# This can be potentially used for cascade attention.
|
||||
num_common_prefix_blocks = 0
|
||||
num_common_prefix_blocks = [0] * len(
|
||||
self.kv_cache_config.kv_cache_groups)
|
||||
if self.running:
|
||||
any_request = self.running[0]
|
||||
num_common_prefix_blocks = (
|
||||
@@ -564,7 +565,7 @@ class Scheduler(SchedulerInterface):
|
||||
request: Request,
|
||||
num_scheduled_tokens: int,
|
||||
num_scheduled_spec_tokens: int,
|
||||
new_block_ids: list[int],
|
||||
new_block_ids: list[list[int]],
|
||||
resumed_from_preemption: bool,
|
||||
) -> CachedRequestData:
|
||||
# OPTIMIZATION: Cache the CachedRequestData objects to avoid creating
|
||||
@@ -939,7 +940,9 @@ class Scheduler(SchedulerInterface):
|
||||
"""
|
||||
if self.connector is None:
|
||||
return False, None
|
||||
block_ids = self.kv_cache_manager.get_block_ids(request.request_id)
|
||||
assert len(self.kv_cache_config.kv_cache_groups
|
||||
) == 1, "KV connector only supports one KV cache group now"
|
||||
block_ids = self.kv_cache_manager.get_block_ids(request.request_id)[0]
|
||||
return self.connector.request_finished(request, block_ids)
|
||||
|
||||
def _update_waiting_for_remote_kv(self, request: Request) -> bool:
|
||||
@@ -956,9 +959,10 @@ class Scheduler(SchedulerInterface):
|
||||
"""
|
||||
if request.request_id not in self.finished_recving_kv_req_ids:
|
||||
return False
|
||||
|
||||
assert len(self.kv_cache_config.kv_cache_groups
|
||||
) == 1, "KV connector only supports one KV cache group now"
|
||||
# Now that the blocks are ready, actually cache them.
|
||||
block_ids = self.kv_cache_manager.get_block_ids(request.request_id)
|
||||
block_ids = self.kv_cache_manager.get_block_ids(request.request_id)[0]
|
||||
num_computed_tokens = len(block_ids) * self.block_size
|
||||
if num_computed_tokens == request.num_tokens:
|
||||
num_computed_tokens -= 1
|
||||
|
||||
Reference in New Issue
Block a user