Adds method to read the pooling types from model's files (#9506)
Signed-off-by: Flavia Beo <flavia.beo@ibm.com> Signed-off-by: Max de Bayser <mbayser@br.ibm.com> Co-authored-by: Max de Bayser <mbayser@br.ibm.com>
This commit is contained in:
co-authored by
Max de Bayser
parent
e036e527a0
commit
aa9078fa03
@@ -25,6 +25,11 @@ def init_tokenizer_from_configs(model_config: ModelConfig,
|
||||
trust_remote_code=model_config.trust_remote_code,
|
||||
revision=model_config.tokenizer_revision)
|
||||
|
||||
if (model_config.encoder_config is not None
|
||||
and "do_lower_case" in model_config.encoder_config):
|
||||
init_kwargs["do_lower_case"] = model_config.encoder_config[
|
||||
"do_lower_case"]
|
||||
|
||||
return get_tokenizer_group(parallel_config.tokenizer_pool_config,
|
||||
**init_kwargs)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user