[Speculative decoding 2/9] Multi-step worker for draft model (#2424)

This commit is contained in:
Cade Daniel
2024-01-21 16:31:47 -08:00
committed by GitHub
parent 71d63ed72e
commit 18bfcdd05c
11 changed files with 658 additions and 12 deletions
+3 -3
View File
@@ -11,7 +11,7 @@ from vllm.model_executor import set_random_seed
from vllm.model_executor.parallel_utils.communication_op import (
broadcast_tensor_dict)
from vllm.model_executor.parallel_utils.parallel_state import (
initialize_model_parallel)
ensure_model_parallel_initialized)
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
from vllm.worker.cache_engine import CacheEngine
from vllm.worker.model_runner import ModelRunner
@@ -227,8 +227,8 @@ def _init_distributed_environment(
# A small all_reduce for warmup.
torch.distributed.all_reduce(torch.zeros(1).cuda())
initialize_model_parallel(parallel_config.tensor_parallel_size,
parallel_config.pipeline_parallel_size)
ensure_model_parallel_initialized(parallel_config.tensor_parallel_size,
parallel_config.pipeline_parallel_size)
def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):