Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -759,9 +759,11 @@ class BlockPool:
|
||||
|
||||
"""
|
||||
for block in blocks:
|
||||
if block.is_null:
|
||||
continue
|
||||
# ref_cnt=0 means this block is in the free list (i.e. eviction
|
||||
# candidate), so remove it.
|
||||
if block.ref_cnt == 0 and not block.is_null:
|
||||
if block.ref_cnt == 0:
|
||||
self.free_block_queue.remove(block)
|
||||
block.ref_cnt += 1
|
||||
if self.metrics_collector:
|
||||
@@ -785,11 +787,13 @@ class BlockPool:
|
||||
blocks_to_evict_first = []
|
||||
other_pools: dict[BlockPool, list[KVCacheBlock]] = {}
|
||||
for block in ordered_blocks:
|
||||
if block.is_null: # never pooled; its ref_cnt is not maintained
|
||||
continue
|
||||
if block.pool is not None and block.pool is not self:
|
||||
other_pools.setdefault(block.pool, []).append(block)
|
||||
continue
|
||||
block.ref_cnt -= 1
|
||||
if block.ref_cnt == 0 and not block.is_null:
|
||||
if block.ref_cnt == 0:
|
||||
if block.block_hash is None or not self.enable_caching:
|
||||
# LIFO reuse of non-cached blocks for better GPU locality.
|
||||
blocks_to_evict_first.append(block)
|
||||
|
||||
@@ -14,7 +14,7 @@ from vllm.v1.core.kv_cache_coordinator import (
|
||||
get_kv_cache_coordinator,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlock, KVCacheBlockCopy
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlock, KVCacheBlockCopy, trim_null_head
|
||||
from vllm.v1.core.single_type_kv_cache_manager import MambaManager
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
@@ -117,7 +117,7 @@ class KVCacheBlocks:
|
||||
return [
|
||||
[
|
||||
block.block_id
|
||||
for block in group
|
||||
for block in trim_null_head(group)
|
||||
if block.block_hash is None and not block.is_null
|
||||
]
|
||||
for group in self.blocks
|
||||
|
||||
@@ -8,7 +8,15 @@ import math
|
||||
import os
|
||||
from collections.abc import Callable, Iterable, Iterator, Sequence
|
||||
from dataclasses import dataclass, field, replace
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple, NewType, TypeAlias, overload
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
NamedTuple,
|
||||
NewType,
|
||||
TypeAlias,
|
||||
TypeVar,
|
||||
overload,
|
||||
)
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config import VllmConfig
|
||||
@@ -216,6 +224,32 @@ class KVCacheBlock:
|
||||
)
|
||||
|
||||
|
||||
def null_head_len(blocks: Sequence[KVCacheBlock]) -> int:
|
||||
"""Number of leading null blocks, e.g. a sliding-window table's skipped prefix."""
|
||||
if not blocks or not blocks[0].is_null:
|
||||
return 0
|
||||
lo, hi = 1, len(blocks)
|
||||
while lo < hi:
|
||||
mid = (lo + hi) // 2
|
||||
if blocks[mid].is_null:
|
||||
lo = mid + 1
|
||||
else:
|
||||
hi = mid
|
||||
# All null entries are one shared object, so count() checks the run at C speed.
|
||||
if blocks[:lo].count(blocks[0]) == lo:
|
||||
return lo
|
||||
return next(i for i, b in enumerate(blocks) if not b.is_null)
|
||||
|
||||
|
||||
_BlockSeq = TypeVar("_BlockSeq", list[KVCacheBlock], Sequence[KVCacheBlock])
|
||||
|
||||
|
||||
def trim_null_head(blocks: _BlockSeq) -> _BlockSeq:
|
||||
"""``blocks`` without its leading null blocks; ``blocks`` itself if none."""
|
||||
nhl = null_head_len(blocks)
|
||||
return blocks[nhl:] if nhl else blocks
|
||||
|
||||
|
||||
class KVCacheBlockCopy(NamedTuple):
|
||||
src_block_id: int
|
||||
dst_block_id: int
|
||||
|
||||
@@ -60,14 +60,16 @@ class AsyncScheduler(Scheduler):
|
||||
self, request: Request, new_token_ids: list[int], is_stale: bool = False
|
||||
) -> tuple[list[int], bool]:
|
||||
status_before_update = request.status
|
||||
# Tokens trimmed at a stop resolve their placeholders too.
|
||||
num_new_tokens = len(new_token_ids)
|
||||
new_token_ids, stopped = super()._update_request_with_output(
|
||||
request, new_token_ids
|
||||
request, new_token_ids, is_stale
|
||||
)
|
||||
|
||||
# Placeholders were zeroed at preemption; a stale delivery must not
|
||||
# decrement them (it would underflow).
|
||||
if not is_stale:
|
||||
request.num_output_placeholders -= len(new_token_ids)
|
||||
request.num_output_placeholders -= num_new_tokens
|
||||
assert request.num_output_placeholders >= 0
|
||||
|
||||
# Cache the new tokens. Preempted requests should be skipped.
|
||||
|
||||
@@ -368,6 +368,36 @@ class Scheduler(SchedulerInterface):
|
||||
# prefill batch fully drained the waiting queue. Prefill throttling
|
||||
# is disabled in this case.
|
||||
self.prefill_capacity_bound = False
|
||||
# The DP engine core drives this cadence from a counter shared across
|
||||
# ranks; an engine without one never throttles, so apply the interval
|
||||
# locally, timed from the last prefill-carrying step. Three regimes:
|
||||
# 1. no DP -- local cadence applies;
|
||||
# 2. dense DP rank -- `reconfigure_for_independent_dp_rank` already
|
||||
# set data_parallel_size=1, so each rank gets its own cadence.
|
||||
# Intended, but a behaviour change: this used to be a no-op;
|
||||
# 3. MoE DP rank (size > 1) -- the engine core drives it, no-op here.
|
||||
self.local_prefill_interval = (
|
||||
self.scheduler_config.prefill_schedule_interval
|
||||
if self.parallel_config.data_parallel_size == 1
|
||||
else 1
|
||||
)
|
||||
if self.local_prefill_interval > 1:
|
||||
# Neither the hold nor its release is otherwise visible in logs or
|
||||
# stats, which makes a misapplied cadence hard to diagnose.
|
||||
logger.info_once(
|
||||
"Prefill cadence: applying prefill_schedule_interval=%d "
|
||||
"locally (this engine is not DP-coordinated); prefill is held "
|
||||
"until %d executed steps have passed since the last "
|
||||
"prefill-carrying step.",
|
||||
self.local_prefill_interval,
|
||||
self.local_prefill_interval,
|
||||
)
|
||||
# A full interval back, so the first step is never gated.
|
||||
self.last_prefill_step = -self.local_prefill_interval
|
||||
# Steps that actually scheduled tokens. `current_step` counts every
|
||||
# schedule() call including empty ones, so it would expire the hold
|
||||
# early; it keeps its own meaning (next_decode_eligible_step).
|
||||
self.num_executed_steps = 0
|
||||
self.scheduler_reserve_full_isl = (
|
||||
self.scheduler_config.scheduler_reserve_full_isl
|
||||
)
|
||||
@@ -623,6 +653,9 @@ class Scheduler(SchedulerInterface):
|
||||
scheduled_spec_decode_tokens: dict[str, list[int]] = {}
|
||||
# Whether the running batch contains any prefill requests.
|
||||
prefill_scheduled = False
|
||||
# Whether a waiting request was admitted with prompt left to compute.
|
||||
# Separate from `prefill_scheduled`, which covers only the running loop.
|
||||
prefill_admitted = False
|
||||
# Whether any scheduled request has a synchronous connector KV load.
|
||||
has_sync_kv_loads = False
|
||||
|
||||
@@ -654,6 +687,28 @@ class Scheduler(SchedulerInterface):
|
||||
long_prefill_token_threshold, input_budget // num_eligible_reqs
|
||||
)
|
||||
|
||||
# Local prefill cadence: hold prefill until the interval since the last
|
||||
# prefill-carrying step has elapsed, so decode runs in longer
|
||||
# uninterrupted stretches. Mirrors the DP gate above, including its
|
||||
# capacity escape: once a release step cannot drain the waiting queue,
|
||||
# throttling stops (#44558, test_throttle_capacity_bound_guard_admits).
|
||||
# Without that escape a backed-up engine holds both admissions and
|
||||
# in-flight chunks for interval-1 of every interval steps, capping
|
||||
# prefill at roughly one budget per interval and growing the waiting
|
||||
# queue without bound.
|
||||
if (
|
||||
not defer_prefills
|
||||
and self.local_prefill_interval > 1
|
||||
and not self.prefill_capacity_bound
|
||||
# This call becomes executed step num_executed_steps + 1 if it
|
||||
# schedules anything, and last_prefill_step is stamped on the same
|
||||
# convention below.
|
||||
and (self.num_executed_steps + 1) - self.last_prefill_step
|
||||
< self.local_prefill_interval
|
||||
and any(not r.is_prefill_chunk for r in self.running)
|
||||
):
|
||||
defer_prefills = True
|
||||
|
||||
# First, schedule the RUNNING requests.
|
||||
req_index = 0
|
||||
while req_index < len(self.running) and token_budget > 0:
|
||||
@@ -1318,6 +1373,12 @@ class Scheduler(SchedulerInterface):
|
||||
)
|
||||
|
||||
if load_kv_async:
|
||||
# This path deliberately does not reach `prefill_admitted`
|
||||
# below, so it does not restart the prefill cadence:
|
||||
# `num_new_tokens` was set to 0 above, so the step carries
|
||||
# no prefill compute for this request. The interval is
|
||||
# timed from steps that actually prefilled; refreshing it
|
||||
# here would start the cadence from a step that did none.
|
||||
# If loading async, allocate memory and put request
|
||||
# into the WAITING_FOR_REMOTE_KV state.
|
||||
request.status = RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
@@ -1373,6 +1434,14 @@ class Scheduler(SchedulerInterface):
|
||||
scheduled_spec_decode_tokens[request_id] = [
|
||||
-1
|
||||
] * self.num_spec_tokens
|
||||
# Must use the SAME predicate as the deferral gate in the
|
||||
# waiting loop (`num_computed_tokens < num_tokens - 1`).
|
||||
# Stamping on `< num_prompt_tokens` instead let a request with
|
||||
# exactly one token left pass the gate on a held step and still
|
||||
# restart the interval -- which is every full prefix-cache hit,
|
||||
# and every remote-KV resume, since those set
|
||||
# num_computed_tokens = num_tokens - 1.
|
||||
prefill_admitted |= num_computed_tokens < request.num_tokens - 1
|
||||
# Only track requests that will still be prefilling after this chunk.
|
||||
if num_computed_tokens + num_new_tokens < request.num_tokens:
|
||||
self._inflight_prefills.add(request)
|
||||
@@ -1403,6 +1472,15 @@ class Scheduler(SchedulerInterface):
|
||||
# Check if the scheduling constraints are satisfied.
|
||||
total_num_scheduled_tokens = sum(num_scheduled_tokens.values())
|
||||
assert total_num_scheduled_tokens <= self.max_num_scheduled_tokens
|
||||
# Count this step before stamping the cadence below, so that
|
||||
# last_prefill_step includes the step that carried the prefill.
|
||||
if total_num_scheduled_tokens > 0:
|
||||
self.num_executed_steps += 1
|
||||
|
||||
# Restart the cadence interval from any step that carried prefill,
|
||||
# whether a new admission or a running chunk.
|
||||
if self.local_prefill_interval > 1 and (prefill_scheduled or prefill_admitted):
|
||||
self.last_prefill_step = self.num_executed_steps
|
||||
|
||||
assert token_budget >= 0
|
||||
assert input_budget >= 0
|
||||
@@ -2496,7 +2574,6 @@ class Scheduler(SchedulerInterface):
|
||||
def _update_request_with_output(
|
||||
self, request: Request, new_token_ids: list[int], is_stale: bool = False
|
||||
) -> tuple[list[int], bool]:
|
||||
# is_stale is only used by the AsyncScheduler override.
|
||||
# Append generated tokens and check for stop. Note that if
|
||||
# a request is still being prefilled, we expect the model runner
|
||||
# to return empty token ids for the request.
|
||||
@@ -2508,6 +2585,10 @@ class Scheduler(SchedulerInterface):
|
||||
# This must be called before we make the EngineCoreOutput.
|
||||
stopped = check_stop(request, self.max_model_len)
|
||||
if stopped:
|
||||
# num_computed_tokens covers all new tokens but the last one.
|
||||
# Keep that true after trimming: resumable requests resume from it.
|
||||
if not is_stale:
|
||||
request.num_computed_tokens -= len(new_token_ids) - num_new
|
||||
del new_token_ids[num_new:] # Trim new tokens if needed.
|
||||
break
|
||||
return new_token_ids, stopped
|
||||
|
||||
@@ -16,6 +16,7 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHashWithGroupId,
|
||||
KVCacheBlock,
|
||||
resolve_block_hashes,
|
||||
trim_null_head,
|
||||
)
|
||||
from vllm.v1.hisparse.block_pool import SharedEventQueueBlockPool
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
@@ -569,14 +570,15 @@ class SingleTypeKVCacheManager(ABC):
|
||||
request_id: The request ID.
|
||||
|
||||
Returns:
|
||||
The request's blocks in allocation order.
|
||||
The request's blocks in allocation order, without the leading
|
||||
null blocks (e.g. a sliding-window table's skipped prefix).
|
||||
|
||||
"""
|
||||
# Default to [] in case a request is freed (aborted) before alloc.
|
||||
req_blocks = self.req_to_blocks.pop(request_id, [])
|
||||
self.num_cached_block.pop(request_id, None)
|
||||
self._partial_hit_reqs.pop(request_id, None)
|
||||
return req_blocks
|
||||
return trim_null_head(req_blocks)
|
||||
|
||||
def free(self, request_id: str) -> None:
|
||||
"""Free the blocks for the request.
|
||||
@@ -2547,7 +2549,7 @@ class HiSparseResidentManager(_HiSparseAuxiliaryManager):
|
||||
blocks = self.get_residency_row(request_id)
|
||||
self.residency_changes.pop(request_id, None)
|
||||
super().pop_blocks_for_free(request_id)
|
||||
return blocks
|
||||
return trim_null_head(blocks)
|
||||
|
||||
def adopt_resident_page(
|
||||
self, request_id: str, block_idx: int, block: KVCacheBlock
|
||||
|
||||
Reference in New Issue
Block a user