treat do_lower_case in the same way as the sentence-transformers library (#11815)

Signed-off-by: Max de Bayser <mbayser@br.ibm.com>
This commit is contained in:
Maximilien de Bayser
2025-01-09 11:05:43 +08:00
committed by GitHub
parent 615e4a5401
commit 1fe554bac3
5 changed files with 13 additions and 5 deletions
@@ -26,11 +26,6 @@ def init_tokenizer_from_configs(model_config: ModelConfig,
trust_remote_code=model_config.trust_remote_code,
revision=model_config.tokenizer_revision)
if (model_config.encoder_config is not None
and "do_lower_case" in model_config.encoder_config):
init_kwargs["do_lower_case"] = model_config.encoder_config[
"do_lower_case"]
return get_tokenizer_group(parallel_config.tokenizer_pool_config,
**init_kwargs)