[Bug fix][Core] assert num_new_tokens == 1 fails when SamplingParams.n is not 1 and max_tokens is large & Add tests for preemption (#4451)

This commit is contained in:
SangBin Cho
2024-05-01 19:24:13 -07:00
committed by GitHub
parent b8afa8b95a
commit 0d62fe58db
6 changed files with 172 additions and 13 deletions
+2 -1
View File
@@ -296,6 +296,7 @@ class VllmRunner:
tensor_parallel_size: int = 1,
block_size: int = 16,
enable_chunked_prefill: bool = False,
swap_space=4,
**kwargs,
) -> None:
self.model = LLM(
@@ -303,7 +304,7 @@ class VllmRunner:
tokenizer=tokenizer_name,
trust_remote_code=True,
dtype=dtype,
swap_space=0,
swap_space=swap_space,
disable_log_stats=disable_log_stats,
tensor_parallel_size=tensor_parallel_size,
max_model_len=max_model_len,