[2/N][Core] support partial prefix cache hit for hybrid model (#46384)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
co-authored by
Yifan Qiao
parent
8e981630c9
commit
481e481be7
@@ -112,7 +112,7 @@ from vllm.v1.worker.gpu.spec_decode.utils import DraftTokensHandler
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
from vllm.v1.worker.gpu.structured_outputs import StructuredOutputsWorker
|
||||
from vllm.v1.worker.lora_model_runner_mixin import LoRAModelRunnerMixin
|
||||
from vllm.v1.worker.utils import KVBlockZeroer
|
||||
from vllm.v1.worker.utils import KVBlockZeroer, copy_kv_cache_blocks_inplace
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -838,6 +838,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
assert self.kv_block_zeroer is not None
|
||||
self.kv_block_zeroer.zero_block_ids(scheduler_output.new_block_ids_to_zero)
|
||||
|
||||
# Apply copy-on-write block copies for partial prefix-cache hits, after
|
||||
# zeroing new blocks and before the forward pass reads them.
|
||||
if scheduler_output.kv_cache_block_copies:
|
||||
copy_kv_cache_blocks_inplace(
|
||||
self.kv_caches,
|
||||
self.kv_cache_config.num_blocks,
|
||||
scheduler_output.kv_cache_block_copies,
|
||||
)
|
||||
|
||||
def prepare_inputs(
|
||||
self, scheduler_output: SchedulerOutput, batch_desc: BatchExecutionDescriptor
|
||||
) -> InputBatch:
|
||||
|
||||
Reference in New Issue
Block a user