Asynchronous tokenization (#2879)
This commit is contained in:
@@ -3,6 +3,7 @@ from dataclasses import dataclass
|
||||
import os
|
||||
from packaging.version import Version
|
||||
|
||||
import json
|
||||
import torch
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
@@ -389,6 +390,58 @@ class CacheConfig:
|
||||
logger.warning("Possibly too large swap space. " + msg)
|
||||
|
||||
|
||||
@dataclass
|
||||
class TokenizerPoolConfig:
|
||||
"""Configuration for the tokenizer pool.
|
||||
|
||||
Args:
|
||||
pool_size: Number of tokenizer workers in the pool.
|
||||
pool_type: Type of the pool.
|
||||
extra_config: Additional config for the pool.
|
||||
The way the config will be used depends on the
|
||||
pool type.
|
||||
"""
|
||||
pool_size: int
|
||||
pool_type: str
|
||||
extra_config: dict
|
||||
|
||||
def __post_init__(self):
|
||||
if self.pool_type not in ("ray", ):
|
||||
raise ValueError(f"Unknown pool type: {self.pool_type}")
|
||||
if not isinstance(self.extra_config, dict):
|
||||
raise ValueError("extra_config must be a dictionary.")
|
||||
|
||||
@classmethod
|
||||
def create_config(
|
||||
cls, tokenizer_pool_size: int, tokenizer_pool_type: str,
|
||||
tokenizer_pool_extra_config: Optional[Union[str, dict]]
|
||||
) -> Optional["TokenizerPoolConfig"]:
|
||||
"""Create a TokenizerPoolConfig from the given parameters.
|
||||
|
||||
If tokenizer_pool_size is 0, return None.
|
||||
|
||||
Args:
|
||||
tokenizer_pool_size: Number of tokenizer workers in the pool.
|
||||
tokenizer_pool_type: Type of the pool.
|
||||
tokenizer_pool_extra_config: Additional config for the pool.
|
||||
The way the config will be used depends on the
|
||||
pool type. This can be a JSON string (will be parsed).
|
||||
"""
|
||||
if tokenizer_pool_size:
|
||||
if isinstance(tokenizer_pool_extra_config, str):
|
||||
tokenizer_pool_extra_config_parsed = json.loads(
|
||||
tokenizer_pool_extra_config)
|
||||
else:
|
||||
tokenizer_pool_extra_config_parsed = (
|
||||
tokenizer_pool_extra_config or {})
|
||||
tokenizer_pool_config = cls(tokenizer_pool_size,
|
||||
tokenizer_pool_type,
|
||||
tokenizer_pool_extra_config_parsed)
|
||||
else:
|
||||
tokenizer_pool_config = None
|
||||
return tokenizer_pool_config
|
||||
|
||||
|
||||
class ParallelConfig:
|
||||
"""Configuration for the distributed execution.
|
||||
|
||||
@@ -403,6 +456,8 @@ class ParallelConfig:
|
||||
parallel and large models.
|
||||
disable_custom_all_reduce: Disable the custom all-reduce kernel and
|
||||
fall back to NCCL.
|
||||
tokenizer_pool_config: Config for the tokenizer pool.
|
||||
If None, will use synchronous tokenization.
|
||||
ray_workers_use_nsight: Whether to profile Ray workers with nsight, see
|
||||
https://docs.ray.io/en/latest/ray-observability/user-guides/profiling.html#profiling-nsight-profiler.
|
||||
"""
|
||||
@@ -414,6 +469,7 @@ class ParallelConfig:
|
||||
worker_use_ray: bool,
|
||||
max_parallel_loading_workers: Optional[int] = None,
|
||||
disable_custom_all_reduce: bool = False,
|
||||
tokenizer_pool_config: Optional[TokenizerPoolConfig] = None,
|
||||
ray_workers_use_nsight: bool = False,
|
||||
placement_group: Optional["PlacementGroup"] = None,
|
||||
) -> None:
|
||||
@@ -430,6 +486,7 @@ class ParallelConfig:
|
||||
self.worker_use_ray = worker_use_ray
|
||||
self.max_parallel_loading_workers = max_parallel_loading_workers
|
||||
self.disable_custom_all_reduce = disable_custom_all_reduce
|
||||
self.tokenizer_pool_config = tokenizer_pool_config
|
||||
self.ray_workers_use_nsight = ray_workers_use_nsight
|
||||
self.placement_group = placement_group
|
||||
|
||||
|
||||
Reference in New Issue
Block a user