[Fix] Don't deep-copy LogitsProcessors when copying SamplingParams (#3099)

This commit is contained in:
Nick Hill
2024-02-29 19:20:42 +00:00
committed by GitHub
parent 2c08ff23c0
commit 29a8d6a554
2 changed files with 18 additions and 2 deletions
+3 -2
View File
@@ -484,8 +484,9 @@ class LLMEngine:
prompt_token_ids[:prefix_pos], lora_request.lora_int_id
if lora_request else 0) if prefix_pos is not None else None
# Defensive copy of SamplingParams, which are used by the sampler
sampling_params = copy.deepcopy(sampling_params)
# Defensive copy of SamplingParams, which are used by the sampler,
# this doesn't deep-copy LogitsProcessor objects
sampling_params = sampling_params.clone()
# Create the sequence group.
seq_group = SequenceGroup(request_id, [seq], sampling_params,