[kv_offload] Skip decode-phase blocks in CPU offload (#43797)
Signed-off-by: Itay Etelis <itay.etelis@ibm.com> Co-authored-by: Itay Etelis <itay.etelis@ibm.com>
This commit is contained in:
co-authored by
Itay Etelis
parent
9636709372
commit
d63108fb18
@@ -94,6 +94,7 @@ class SchedulerOffloadConfig(NamedTuple):
|
||||
kv_group_configs: tuple[GroupOffloadConfig, ...]
|
||||
block_size_factor: int
|
||||
num_workers: int
|
||||
offload_prompt_only: bool
|
||||
|
||||
@classmethod
|
||||
def from_spec(cls, spec: OffloadingSpec) -> "SchedulerOffloadConfig":
|
||||
@@ -156,6 +157,7 @@ class SchedulerOffloadConfig(NamedTuple):
|
||||
for idx, gpu_block_size in enumerate(spec.gpu_block_size)
|
||||
),
|
||||
block_size_factor=spec.block_size_factor,
|
||||
offload_prompt_only=spec.offload_prompt_only,
|
||||
)
|
||||
|
||||
|
||||
@@ -710,6 +712,15 @@ class OffloadingConnectorScheduler:
|
||||
if max_offload_tokens is not None:
|
||||
num_offloadable_tokens = min(num_offloadable_tokens, max_offload_tokens)
|
||||
|
||||
# Skip decode-phase blocks: clamp to the prompt length so only
|
||||
# prefill (prompt) blocks become eligible for store. next_stored_idx
|
||||
# never advances past this boundary, so decode blocks are never
|
||||
# queued in this or any later step.
|
||||
if self.config.offload_prompt_only:
|
||||
num_offloadable_tokens = min(
|
||||
num_offloadable_tokens, req.num_prompt_tokens
|
||||
)
|
||||
|
||||
# Filter out blocks skipped due to sliding window attention / SSM
|
||||
# or unreachable by the load path's alignment constraints.
|
||||
new_offload_keys: list[OffloadKey] = []
|
||||
|
||||
@@ -380,6 +380,14 @@ class OffloadingSpec(ABC):
|
||||
assert kv_transfer_config is not None
|
||||
self.extra_config = kv_transfer_config.kv_connector_extra_config
|
||||
|
||||
# When True, only prompt (prefill) blocks are offloaded; decode-phase
|
||||
# blocks (KV generated after the prompt) are skipped. Useful when prior
|
||||
# turns' generated tokens are dropped before the next turn (e.g.
|
||||
# reasoning models that strip thinking).
|
||||
self.offload_prompt_only: bool = bool(
|
||||
self.extra_config.get("offload_prompt_only", True)
|
||||
)
|
||||
|
||||
parallel_config = vllm_config.parallel_config
|
||||
context_parallel_factor = (
|
||||
parallel_config.decode_context_parallel_size
|
||||
|
||||
Reference in New Issue
Block a user