Asynchronous tokenization (#2879)
This commit is contained in:
@@ -4,7 +4,8 @@ from dataclasses import dataclass
|
||||
from typing import Optional, Tuple
|
||||
|
||||
from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
|
||||
ParallelConfig, SchedulerConfig, LoRAConfig)
|
||||
ParallelConfig, SchedulerConfig, LoRAConfig,
|
||||
TokenizerPoolConfig)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -40,6 +41,9 @@ class EngineArgs:
|
||||
enforce_eager: bool = False
|
||||
max_context_len_to_capture: int = 8192
|
||||
disable_custom_all_reduce: bool = False
|
||||
tokenizer_pool_size: int = 0
|
||||
tokenizer_pool_type: str = "ray"
|
||||
tokenizer_pool_extra_config: Optional[dict] = None
|
||||
enable_lora: bool = False
|
||||
max_loras: int = 1
|
||||
max_lora_rank: int = 16
|
||||
@@ -249,6 +253,25 @@ class EngineArgs:
|
||||
action='store_true',
|
||||
default=EngineArgs.disable_custom_all_reduce,
|
||||
help='See ParallelConfig')
|
||||
parser.add_argument('--tokenizer-pool-size',
|
||||
type=int,
|
||||
default=EngineArgs.tokenizer_pool_size,
|
||||
help='Size of tokenizer pool to use for '
|
||||
'asynchronous tokenization. If 0, will '
|
||||
'use synchronous tokenization.')
|
||||
parser.add_argument('--tokenizer-pool-type',
|
||||
type=str,
|
||||
default=EngineArgs.tokenizer_pool_type,
|
||||
help='Type of tokenizer pool to use for '
|
||||
'asynchronous tokenization. Ignored '
|
||||
'if tokenizer_pool_size is 0.')
|
||||
parser.add_argument('--tokenizer-pool-extra-config',
|
||||
type=str,
|
||||
default=EngineArgs.tokenizer_pool_extra_config,
|
||||
help='Extra config for tokenizer pool. '
|
||||
'This should be a JSON string that will be '
|
||||
'parsed into a dictionary. Ignored if '
|
||||
'tokenizer_pool_size is 0.')
|
||||
# LoRA related configs
|
||||
parser.add_argument('--enable-lora',
|
||||
action='store_true',
|
||||
@@ -312,14 +335,16 @@ class EngineArgs:
|
||||
cache_config = CacheConfig(self.block_size,
|
||||
self.gpu_memory_utilization,
|
||||
self.swap_space, self.kv_cache_dtype,
|
||||
model_config.get_sliding_window(),
|
||||
self.enable_prefix_caching)
|
||||
parallel_config = ParallelConfig(self.pipeline_parallel_size,
|
||||
self.tensor_parallel_size,
|
||||
self.worker_use_ray,
|
||||
self.max_parallel_loading_workers,
|
||||
self.disable_custom_all_reduce,
|
||||
self.ray_workers_use_nsight)
|
||||
model_config.get_sliding_window())
|
||||
parallel_config = ParallelConfig(
|
||||
self.pipeline_parallel_size, self.tensor_parallel_size,
|
||||
self.worker_use_ray, self.max_parallel_loading_workers,
|
||||
self.disable_custom_all_reduce,
|
||||
TokenizerPoolConfig.create_config(
|
||||
self.tokenizer_pool_size,
|
||||
self.tokenizer_pool_type,
|
||||
self.tokenizer_pool_extra_config,
|
||||
), self.ray_workers_use_nsight)
|
||||
scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
|
||||
self.max_num_seqs,
|
||||
model_config.max_model_len,
|
||||
|
||||
@@ -17,8 +17,9 @@ from vllm.outputs import RequestOutput
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.sequence import (Logprob, SamplerOutput, Sequence, SequenceGroup,
|
||||
SequenceGroupOutput, SequenceOutput, SequenceStatus)
|
||||
from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
|
||||
TokenizerGroup)
|
||||
from vllm.transformers_utils.tokenizer import detokenize_incrementally
|
||||
from vllm.transformers_utils.tokenizer_group import (BaseTokenizerGroup,
|
||||
get_tokenizer_group)
|
||||
from vllm.utils import Counter
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -102,6 +103,10 @@ class LLMEngine:
|
||||
parallel_config, scheduler_config,
|
||||
device_config, lora_config)
|
||||
|
||||
# Ping the tokenizer to ensure liveness if it runs in a
|
||||
# different process.
|
||||
self.tokenizer.ping()
|
||||
|
||||
# Create the scheduler.
|
||||
# NOTE: the cache_config here have been updated with the numbers of
|
||||
# GPU and CPU blocks, which are profiled in the distributed executor.
|
||||
@@ -152,6 +157,7 @@ class LLMEngine:
|
||||
|
||||
def _init_tokenizer(self, **tokenizer_init_kwargs):
|
||||
init_kwargs = dict(
|
||||
tokenizer_id=self.model_config.tokenizer,
|
||||
enable_lora=bool(self.lora_config),
|
||||
max_num_seqs=self.scheduler_config.max_num_seqs,
|
||||
max_input_length=None,
|
||||
@@ -159,8 +165,9 @@ class LLMEngine:
|
||||
trust_remote_code=self.model_config.trust_remote_code,
|
||||
revision=self.model_config.tokenizer_revision)
|
||||
init_kwargs.update(tokenizer_init_kwargs)
|
||||
self.tokenizer: TokenizerGroup = TokenizerGroup(
|
||||
self.model_config.tokenizer, **init_kwargs)
|
||||
|
||||
self.tokenizer: BaseTokenizerGroup = get_tokenizer_group(
|
||||
self.parallel_config.tokenizer_pool_config, **init_kwargs)
|
||||
|
||||
def _verify_args(self) -> None:
|
||||
self.model_config.verify_with_parallel_config(self.parallel_config)
|
||||
|
||||
Reference in New Issue
Block a user