[Speculative decoding 6/9] Integrate speculative decoding with LLMEngine (#3894)
This commit is contained in:
@@ -48,10 +48,13 @@ class NeuronExecutor(ExecutorBase):
|
||||
seq_group_metadata_list: List[SequenceGroupMetadata],
|
||||
blocks_to_swap_in: Dict[int, int],
|
||||
blocks_to_swap_out: Dict[int, int],
|
||||
blocks_to_copy: Dict[int, List[int]]) -> SamplerOutput:
|
||||
blocks_to_copy: Dict[int, List[int]],
|
||||
num_lookahead_slots: int) -> List[SamplerOutput]:
|
||||
assert (blocks_to_swap_in == {} and blocks_to_swap_out == {}
|
||||
and blocks_to_copy == {}), (
|
||||
"Cache operations are not supported for Neuron backend.")
|
||||
assert num_lookahead_slots == 0, (
|
||||
"lookahead not supported for Neuron backend.")
|
||||
|
||||
output = self.driver_worker.execute_model(
|
||||
seq_group_metadata_list=seq_group_metadata_list)
|
||||
|
||||
Reference in New Issue
Block a user