[Bugfix] Fix broken OpenAI tensorizer test (#8258)

This commit is contained in:
Cyrus Leung
2024-09-07 08:02:39 +00:00
committed by GitHub
parent ce2702a923
commit 9f68e00d27
4 changed files with 81 additions and 40 deletions
@@ -99,6 +99,13 @@ class TensorizerConfig:
"Loading a model using Tensorizer with quantization on vLLM"
" is unstable and may lead to errors.")
def open_stream(self, tensorizer_args: Optional["TensorizerArgs"] = None):
if tensorizer_args is None:
tensorizer_args = self._construct_tensorizer_args()
return open_stream(self.tensorizer_uri,
**tensorizer_args.stream_params)
def load_with_tensorizer(tensorizer_config: TensorizerConfig,
**extra_kwargs) -> nn.Module: