[2/N][Core] support partial prefix cache hit for hybrid model (#46384)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Jiangyun Zhu
2026-07-12 05:37:51 +00:00
committed by GitHub
co-authored by Yifan Qiao
parent 8e981630c9
commit 481e481be7
21 changed files with 1653 additions and 177 deletions
+10 -1
View File
@@ -112,7 +112,7 @@ from vllm.v1.worker.gpu.spec_decode.utils import DraftTokensHandler
from vllm.v1.worker.gpu.states import RequestState
from vllm.v1.worker.gpu.structured_outputs import StructuredOutputsWorker
from vllm.v1.worker.lora_model_runner_mixin import LoRAModelRunnerMixin
from vllm.v1.worker.utils import KVBlockZeroer
from vllm.v1.worker.utils import KVBlockZeroer, copy_kv_cache_blocks_inplace
logger = init_logger(__name__)
@@ -838,6 +838,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
assert self.kv_block_zeroer is not None
self.kv_block_zeroer.zero_block_ids(scheduler_output.new_block_ids_to_zero)
# Apply copy-on-write block copies for partial prefix-cache hits, after
# zeroing new blocks and before the forward pass reads them.
if scheduler_output.kv_cache_block_copies:
copy_kv_cache_blocks_inplace(
self.kv_caches,
self.kv_cache_config.num_blocks,
scheduler_output.kv_cache_block_copies,
)
def prepare_inputs(
self, scheduler_output: SchedulerOutput, batch_desc: BatchExecutionDescriptor
) -> InputBatch: