[Bugfix] Prevent cuMemcpyBatchAsync segfault with MTP and KV offloading (#44784)
Signed-off-by: joshua <joshua.abraham@multicorewareinc.com> Co-authored-by: joshua <joshua.abraham@multicorewareinc.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
joshua
Or Ozeri
parent
a7fdfeef72
commit
7ad894c86a
@@ -77,6 +77,10 @@ class GroupOffloadConfig(NamedTuple):
|
||||
# than the MLA full-attention group).
|
||||
# None for full-attention groups or when the optimization doesn't apply.
|
||||
alignment_block_count: int | None = None
|
||||
# True for EAGLE/MTP draft-model attention groups. The trailing block
|
||||
# of these groups is volatile and lacks a stable hash, so it must
|
||||
# be excluded from store and load scheduling.
|
||||
is_eagle_group: bool = False
|
||||
|
||||
|
||||
def get_sliding_window_size_in_blocks(
|
||||
@@ -155,6 +159,27 @@ class SchedulerOffloadConfig(NamedTuple):
|
||||
return None
|
||||
return per_segment
|
||||
|
||||
eagle_groups = {
|
||||
idx
|
||||
for idx, g in enumerate(spec.kv_cache_config.kv_cache_groups)
|
||||
if g.is_eagle_group
|
||||
}
|
||||
|
||||
use_eagle = (
|
||||
spec.vllm_config.speculative_config is not None
|
||||
and spec.vllm_config.speculative_config.use_eagle()
|
||||
)
|
||||
if use_eagle and not eagle_groups:
|
||||
eagle_groups = set(range(len(spec.kv_cache_config.kv_cache_groups)))
|
||||
|
||||
if eagle_groups:
|
||||
logger.info(
|
||||
"KV offloading: EAGLE/MTP draft attention groups %s "
|
||||
"detected. The trailing block of these groups will be "
|
||||
"excluded from offloading due to volatility.",
|
||||
sorted(eagle_groups),
|
||||
)
|
||||
|
||||
return cls(
|
||||
num_workers=spec.vllm_config.parallel_config.world_size,
|
||||
kv_group_configs=tuple(
|
||||
@@ -175,6 +200,7 @@ class SchedulerOffloadConfig(NamedTuple):
|
||||
alignment_block_count=_alignment_block_count(
|
||||
gpu_block_size * spec.block_size_factor, sw
|
||||
),
|
||||
is_eagle_group=idx in eagle_groups,
|
||||
)
|
||||
for idx, gpu_block_size in enumerate(spec.gpu_block_size)
|
||||
),
|
||||
@@ -436,6 +462,11 @@ class OffloadingConnectorScheduler:
|
||||
num_hit_tokens: int = 0
|
||||
defer_lookup = False
|
||||
lookup_groups = self._lookup_groups
|
||||
|
||||
# Tracks which eagle groups have already popped their volatile trailing block
|
||||
# in the current convergence iteration. Reset when a non-eagle group
|
||||
# tightens the hit boundary, requiring a fresh pop.
|
||||
eagle_verified: set[int] = set()
|
||||
while lookup_groups:
|
||||
looked_up_sliding_window: bool = False
|
||||
groups_iter = iter(lookup_groups)
|
||||
@@ -453,6 +484,10 @@ class OffloadingConnectorScheduler:
|
||||
>= req_status.req.num_tokens // offloaded_block_size
|
||||
)
|
||||
|
||||
is_eagle_unverified = (
|
||||
group_config.is_eagle_group and group_idx not in eagle_verified
|
||||
)
|
||||
|
||||
# Constrain to block-aligned boundary for this group
|
||||
max_hit_size_tokens = min(
|
||||
max_hit_size_tokens, len(offload_keys) * offloaded_block_size
|
||||
@@ -461,15 +496,25 @@ class OffloadingConnectorScheduler:
|
||||
# we can only load less than a block, better skip
|
||||
return 0
|
||||
|
||||
num_blocks = min(
|
||||
cdiv(max_hit_size_tokens, offloaded_block_size), len(offload_keys)
|
||||
)
|
||||
start_block_idx = num_computed_tokens // offloaded_block_size
|
||||
offload_keys = offload_keys[start_block_idx:num_blocks]
|
||||
sliding_window_size_in_blocks = (
|
||||
group_config.sliding_window_size_in_blocks
|
||||
)
|
||||
|
||||
# For eagle groups, query one extra block that will be popped.
|
||||
# We only need to increase the query size for sliding window groups.
|
||||
query_max = max_hit_size_tokens
|
||||
if is_eagle_unverified and sliding_window_size_in_blocks is not None:
|
||||
query_max = min(
|
||||
max_hit_size_tokens + offloaded_block_size,
|
||||
len(offload_keys) * offloaded_block_size,
|
||||
)
|
||||
|
||||
num_blocks = min(
|
||||
cdiv(query_max, offloaded_block_size), len(offload_keys)
|
||||
)
|
||||
start_block_idx = num_computed_tokens // offloaded_block_size
|
||||
offload_keys = offload_keys[start_block_idx:num_blocks]
|
||||
|
||||
# end index (in the sliced offload_keys) up to which we
|
||||
# have backend-confirmed hits
|
||||
num_hit_blocks: int | None
|
||||
@@ -478,9 +523,12 @@ class OffloadingConnectorScheduler:
|
||||
offload_keys, req_status.req_context
|
||||
)
|
||||
else:
|
||||
required_window = sliding_window_size_in_blocks
|
||||
if is_eagle_unverified:
|
||||
required_window += 1
|
||||
num_hit_blocks = self._sliding_window_lookup(
|
||||
offload_keys,
|
||||
sliding_window_size_in_blocks,
|
||||
required_window,
|
||||
req_status.req_context,
|
||||
)
|
||||
if num_hit_blocks == 0:
|
||||
@@ -489,6 +537,10 @@ class OffloadingConnectorScheduler:
|
||||
if num_hit_blocks is None:
|
||||
defer_lookup = True
|
||||
else:
|
||||
if is_eagle_unverified:
|
||||
num_hit_blocks -= 1
|
||||
eagle_verified.add(group_idx)
|
||||
|
||||
max_hit_size_tokens = min(
|
||||
max_hit_size_tokens,
|
||||
offloaded_block_size * (start_block_idx + num_hit_blocks),
|
||||
@@ -500,6 +552,8 @@ class OffloadingConnectorScheduler:
|
||||
return 0
|
||||
|
||||
if new_num_hit_tokens < num_hit_tokens:
|
||||
if not group_config.is_eagle_group:
|
||||
eagle_verified.clear()
|
||||
if defer_lookup:
|
||||
# make another iteration on all groups to check
|
||||
# if we still need to defer lookup
|
||||
@@ -791,6 +845,9 @@ class OffloadingConnectorScheduler:
|
||||
self.config.kv_group_configs, req_status.group_states
|
||||
):
|
||||
num_blocks = num_offloadable_tokens // group_config.offloaded_block_size
|
||||
if group_config.is_eagle_group:
|
||||
num_blocks = max(0, num_blocks - 1)
|
||||
|
||||
start_block_idx = group_state.next_stored_block_idx
|
||||
if num_blocks <= start_block_idx:
|
||||
continue
|
||||
|
||||
Reference in New Issue
Block a user