[Test] Add basic correctness test (#2908)

This commit is contained in:
Zhuohan Li
2024-02-18 16:44:50 -08:00
committed by GitHub
parent 537c9755a7
commit a61f0521b8
4 changed files with 91 additions and 2 deletions
+2
View File
@@ -165,6 +165,7 @@ class VllmRunner:
model_name: str,
tokenizer_name: Optional[str] = None,
dtype: str = "half",
tensor_parallel_size: int = 1,
) -> None:
self.model = LLM(
model=model_name,
@@ -172,6 +173,7 @@ class VllmRunner:
trust_remote_code=True,
dtype=dtype,
swap_space=0,
tensor_parallel_size=tensor_parallel_size,
)
def generate(