treat do_lower_case in the same way as the sentence-transformers library (#11815)
Signed-off-by: Max de Bayser <mbayser@br.ibm.com>
This commit is contained in:
@@ -26,11 +26,6 @@ def init_tokenizer_from_configs(model_config: ModelConfig,
|
||||
trust_remote_code=model_config.trust_remote_code,
|
||||
revision=model_config.tokenizer_revision)
|
||||
|
||||
if (model_config.encoder_config is not None
|
||||
and "do_lower_case" in model_config.encoder_config):
|
||||
init_kwargs["do_lower_case"] = model_config.encoder_config[
|
||||
"do_lower_case"]
|
||||
|
||||
return get_tokenizer_group(parallel_config.tokenizer_pool_config,
|
||||
**init_kwargs)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user