[V1] V1 Enablement Oracle (#13726)
Signed-off-by: rshaw@neuralmagic.com <rshaw@neuralmagic.com> Co-authored-by: rshaw@neuralmagic.com <rshaw@neuralmagic.com> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com> Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com> Co-authored-by: Michael Goin <michael@neuralmagic.com>
This commit is contained in:
co-authored by
rshaw@neuralmagic.com <rshaw@neuralmagic.com>
Nicolò Lucchesi
Tyler Michael Smith
Michael Goin
parent
8c0d15d5c5
commit
d4d93db2c5
+17
-10
@@ -1140,6 +1140,10 @@ class CacheConfig:
|
||||
if self.cache_dtype == "auto":
|
||||
pass
|
||||
elif self.cache_dtype in ("fp8", "fp8_e4m3", "fp8_e5m2"):
|
||||
if envs.VLLM_USE_V1:
|
||||
raise NotImplementedError(
|
||||
"V1 does not yet support fp8 KV cache. "
|
||||
"Set VLLM_USE_V1=0 to enable fp8 kv cache.")
|
||||
logger.info(
|
||||
"Using fp8 data type to store kv cache. It reduces the GPU "
|
||||
"memory footprint and boosts the performance. "
|
||||
@@ -3142,16 +3146,7 @@ class CompilationConfig(BaseModel):
|
||||
self.inductor_compile_config[KEY] = False
|
||||
|
||||
if self.splitting_ops is None:
|
||||
if envs.VLLM_USE_V1:
|
||||
# v1 must split the graph on attention ops
|
||||
# for piecewise cudagraph
|
||||
self.splitting_ops = [
|
||||
"vllm.unified_attention",
|
||||
"vllm.unified_attention_with_output",
|
||||
]
|
||||
else:
|
||||
# v0 uses full graph compilation
|
||||
self.splitting_ops = []
|
||||
self.splitting_ops = []
|
||||
|
||||
for k, v in self.inductor_passes.items():
|
||||
if not isinstance(v, str):
|
||||
@@ -3246,6 +3241,15 @@ class CompilationConfig(BaseModel):
|
||||
self.bs_to_padded_graph_size[
|
||||
self.max_capture_size] = self.max_capture_size
|
||||
|
||||
def set_splitting_ops_for_v1(self):
|
||||
# If default, override splitting ops for piecewise cudagraph on V1.
|
||||
# NOTE: this function needs to be called
|
||||
if not self.splitting_ops:
|
||||
self.splitting_ops = [
|
||||
"vllm.unified_attention",
|
||||
"vllm.unified_attention_with_output",
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class VllmConfig:
|
||||
@@ -3297,6 +3301,7 @@ class VllmConfig:
|
||||
vllm_factors: list[Any] = []
|
||||
from vllm import __version__
|
||||
vllm_factors.append(__version__)
|
||||
vllm_factors.append(envs.VLLM_USE_V1)
|
||||
if self.model_config:
|
||||
vllm_factors.append(self.model_config.compute_hash())
|
||||
else:
|
||||
@@ -3460,6 +3465,7 @@ class VllmConfig:
|
||||
# CUDA graphs do not work properly with the custom CUDA kernels.
|
||||
# FIXME(woosuk): Disable inductor to reduce the compilation time
|
||||
# and avoid any potential issues with the inductor.
|
||||
# FIXME(rob): Add function to set all of these.
|
||||
self.compilation_config.custom_ops = ["none"]
|
||||
self.compilation_config.use_cudagraph = True
|
||||
self.compilation_config.use_inductor = True
|
||||
@@ -3467,6 +3473,7 @@ class VllmConfig:
|
||||
self.compilation_config.pass_config.enable_fusion = False
|
||||
self.compilation_config.pass_config.enable_noop = False
|
||||
self.compilation_config.level = CompilationLevel.PIECEWISE
|
||||
self.compilation_config.set_splitting_ops_for_v1()
|
||||
|
||||
self._set_cudagraph_sizes()
|
||||
|
||||
|
||||
+346
-80
@@ -223,15 +223,6 @@ class EngineArgs:
|
||||
if not self.tokenizer:
|
||||
self.tokenizer = self.model
|
||||
|
||||
# Override the default value of enable_prefix_caching if it's not set
|
||||
# by user.
|
||||
if self.enable_prefix_caching is None:
|
||||
self.enable_prefix_caching = bool(envs.VLLM_USE_V1)
|
||||
|
||||
# Override max_num_seqs if it's not set by user.
|
||||
if self.max_num_seqs is None:
|
||||
self.max_num_seqs = 256 if not envs.VLLM_USE_V1 else 1024
|
||||
|
||||
# support `EngineArgs(compilation_config={...})`
|
||||
# without having to manually construct a
|
||||
# CompilationConfig object
|
||||
@@ -246,7 +237,6 @@ class EngineArgs:
|
||||
@staticmethod
|
||||
def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
|
||||
"""Shared CLI arguments for vLLM engine."""
|
||||
|
||||
# Model arguments
|
||||
parser.add_argument(
|
||||
'--model',
|
||||
@@ -1191,24 +1181,51 @@ class EngineArgs:
|
||||
use_tqdm_on_load=self.use_tqdm_on_load,
|
||||
)
|
||||
|
||||
def create_engine_config(self,
|
||||
usage_context: Optional[UsageContext] = None
|
||||
) -> VllmConfig:
|
||||
def create_engine_config(
|
||||
self,
|
||||
usage_context: Optional[UsageContext] = None,
|
||||
) -> VllmConfig:
|
||||
"""
|
||||
Create the VllmConfig.
|
||||
|
||||
NOTE: for autoselection of V0 vs V1 engine, we need to
|
||||
create the ModelConfig first, since ModelConfig's attrs
|
||||
(e.g. the model arch) are needed to make the decision.
|
||||
|
||||
This function set VLLM_USE_V1=X if VLLM_USE_V1 is
|
||||
unspecified by the user.
|
||||
|
||||
If VLLM_USE_V1 is specified by the user but the VllmConfig
|
||||
is incompatible, we raise an error.
|
||||
"""
|
||||
from vllm.platforms import current_platform
|
||||
current_platform.pre_register_and_update()
|
||||
|
||||
if envs.VLLM_USE_V1:
|
||||
self._override_v1_engine_args(usage_context)
|
||||
|
||||
device_config = DeviceConfig(device=self.device)
|
||||
model_config = self.create_model_config()
|
||||
|
||||
if (model_config.is_multimodal_model and not envs.VLLM_USE_V1
|
||||
and self.enable_prefix_caching):
|
||||
logger.warning("--enable-prefix-caching is currently not "
|
||||
"supported for multimodal models in v0 and "
|
||||
"has been disabled.")
|
||||
self.enable_prefix_caching = False
|
||||
# * If VLLM_USE_V1 is unset, we enable V1 for "supported features"
|
||||
# and fall back to V0 for experimental or unsupported features.
|
||||
# * If VLLM_USE_V1=1, we enable V1 for supported + experimental
|
||||
# features and raise error for unsupported features.
|
||||
# * If VLLM_USE_V1=0, we disable V1.
|
||||
use_v1 = False
|
||||
try_v1 = envs.VLLM_USE_V1 or not envs.is_set("VLLM_USE_V1")
|
||||
if try_v1 and self._is_v1_supported_oracle(model_config):
|
||||
use_v1 = True
|
||||
|
||||
# If user explicitly set VLLM_USE_V1, sanity check we respect it.
|
||||
if envs.is_set("VLLM_USE_V1"):
|
||||
assert use_v1 == envs.VLLM_USE_V1
|
||||
# Otherwise, set the VLLM_USE_V1 variable globally.
|
||||
else:
|
||||
envs.set_vllm_use_v1(use_v1)
|
||||
|
||||
# Set default arguments for V0 or V1 Engine.
|
||||
if use_v1:
|
||||
self._set_default_args_v1(usage_context)
|
||||
else:
|
||||
self._set_default_args_v0(model_config)
|
||||
|
||||
cache_config = CacheConfig(
|
||||
block_size=self.block_size,
|
||||
@@ -1239,50 +1256,6 @@ class EngineArgs:
|
||||
worker_extension_cls=self.worker_extension_cls,
|
||||
)
|
||||
|
||||
max_model_len = model_config.max_model_len
|
||||
use_long_context = max_model_len > 32768
|
||||
if self.enable_chunked_prefill is None:
|
||||
# If not explicitly set, enable chunked prefill by default for
|
||||
# long context (> 32K) models. This is to avoid OOM errors in the
|
||||
# initial memory profiling phase.
|
||||
|
||||
# For multimodal models and models with MLA, chunked prefill is
|
||||
# disabled by default in V0, but enabled by design in V1
|
||||
if model_config.is_multimodal_model or model_config.use_mla:
|
||||
self.enable_chunked_prefill = bool(envs.VLLM_USE_V1)
|
||||
|
||||
elif use_long_context:
|
||||
is_gpu = device_config.device_type == "cuda"
|
||||
use_sliding_window = (model_config.get_sliding_window()
|
||||
is not None)
|
||||
use_spec_decode = self.speculative_model is not None
|
||||
from vllm.platforms import current_platform
|
||||
if (is_gpu and not use_sliding_window and not use_spec_decode
|
||||
and not self.enable_lora
|
||||
and not self.enable_prompt_adapter
|
||||
and model_config.runner_type != "pooling"
|
||||
and not current_platform.is_rocm()):
|
||||
self.enable_chunked_prefill = True
|
||||
logger.warning(
|
||||
"Chunked prefill is enabled by default for models with "
|
||||
"max_model_len > 32K. Currently, chunked prefill might "
|
||||
"not work with some features or models. If you "
|
||||
"encounter any issues, please disable chunked prefill "
|
||||
"by setting --enable-chunked-prefill=False.")
|
||||
if self.enable_chunked_prefill is None:
|
||||
self.enable_chunked_prefill = False
|
||||
|
||||
if not self.enable_chunked_prefill and use_long_context:
|
||||
logger.warning(
|
||||
"The model has a long context length (%s). This may cause OOM "
|
||||
"errors during the initial memory profiling phase, or result "
|
||||
"in low performance due to small KV cache space. Consider "
|
||||
"setting --max-model-len to a smaller value.", max_model_len)
|
||||
elif (self.enable_chunked_prefill
|
||||
and model_config.runner_type == "pooling"):
|
||||
msg = "Chunked prefill is not supported for pooling models"
|
||||
raise ValueError(msg)
|
||||
|
||||
speculative_config = SpeculativeConfig.maybe_create_spec_config(
|
||||
target_model_config=model_config,
|
||||
target_parallel_config=parallel_config,
|
||||
@@ -1425,18 +1398,282 @@ class EngineArgs:
|
||||
additional_config=self.additional_config,
|
||||
)
|
||||
|
||||
if envs.VLLM_USE_V1:
|
||||
self._override_v1_engine_config(config)
|
||||
return config
|
||||
|
||||
def _override_v1_engine_args(self, usage_context: UsageContext) -> None:
|
||||
"""
|
||||
Override the EngineArgs's args based on the usage context for V1.
|
||||
"""
|
||||
assert envs.VLLM_USE_V1, "V1 is not enabled"
|
||||
def _is_v1_supported_oracle(self, model_config: ModelConfig) -> bool:
|
||||
"""Oracle for whether to use V0 or V1 Engine by default."""
|
||||
|
||||
#############################################################
|
||||
# Unsupported Feature Flags on V1.
|
||||
|
||||
if (self.load_format == LoadFormat.TENSORIZER.value
|
||||
or self.load_format == LoadFormat.SHARDED_STATE.value):
|
||||
_raise_or_fallback(
|
||||
feature_name=f"--load_format {self.load_format}",
|
||||
recommend_to_remove=False)
|
||||
return False
|
||||
|
||||
if (self.logits_processor_pattern
|
||||
!= EngineArgs.logits_processor_pattern):
|
||||
_raise_or_fallback(feature_name="--logits-processor-pattern",
|
||||
recommend_to_remove=False)
|
||||
return False
|
||||
|
||||
if self.preemption_mode != EngineArgs.preemption_mode:
|
||||
_raise_or_fallback(feature_name="--preemption-mode",
|
||||
recommend_to_remove=True)
|
||||
return False
|
||||
|
||||
if (self.disable_async_output_proc
|
||||
!= EngineArgs.disable_async_output_proc):
|
||||
_raise_or_fallback(feature_name="--disable-async-output-proc",
|
||||
recommend_to_remove=True)
|
||||
return False
|
||||
|
||||
if self.scheduling_policy != EngineArgs.scheduling_policy:
|
||||
_raise_or_fallback(feature_name="--scheduling-policy",
|
||||
recommend_to_remove=False)
|
||||
return False
|
||||
|
||||
if self.worker_cls != EngineArgs.worker_cls:
|
||||
_raise_or_fallback(feature_name="--worker-cls",
|
||||
recommend_to_remove=False)
|
||||
return False
|
||||
|
||||
if self.worker_extension_cls != EngineArgs.worker_extension_cls:
|
||||
_raise_or_fallback(feature_name="--worker-extension-cls",
|
||||
recommend_to_remove=False)
|
||||
return False
|
||||
|
||||
if self.num_scheduler_steps != EngineArgs.num_scheduler_steps:
|
||||
_raise_or_fallback(feature_name="--num-scheduler-steps",
|
||||
recommend_to_remove=True)
|
||||
return False
|
||||
|
||||
if self.scheduler_delay_factor != EngineArgs.scheduler_delay_factor:
|
||||
_raise_or_fallback(feature_name="--scheduler-delay-factor",
|
||||
recommend_to_remove=True)
|
||||
return False
|
||||
|
||||
if self.additional_config != EngineArgs.additional_config:
|
||||
_raise_or_fallback(feature_name="--additional-config",
|
||||
recommend_to_remove=False)
|
||||
return False
|
||||
|
||||
# Only support Xgrammar for guided decoding so far.
|
||||
SUPPORTED_GUIDED_DECODING = ["xgrammar", "xgrammar:nofallback"]
|
||||
if self.guided_decoding_backend not in SUPPORTED_GUIDED_DECODING:
|
||||
_raise_or_fallback(feature_name="--guided-decoding-backend",
|
||||
recommend_to_remove=False)
|
||||
return False
|
||||
|
||||
# Need at least Ampere for now (FA support required).
|
||||
from vllm.platforms import current_platform
|
||||
if (current_platform.is_cuda()
|
||||
and current_platform.get_device_capability().major < 8):
|
||||
_raise_or_fallback(feature_name="Compute Capability < 8.0",
|
||||
recommend_to_remove=False)
|
||||
return False
|
||||
|
||||
# No Fp8 KV cache so far.
|
||||
if self.kv_cache_dtype != "auto":
|
||||
_raise_or_fallback(feature_name="--kv-cache-dtype",
|
||||
recommend_to_remove=False)
|
||||
return False
|
||||
|
||||
# No Prompt Adapter so far.
|
||||
if self.enable_prompt_adapter:
|
||||
_raise_or_fallback(feature_name="--enable-prompt-adapter",
|
||||
recommend_to_remove=False)
|
||||
return False
|
||||
|
||||
# No MistralTokenizer support so far (not compatible
|
||||
# with xgrammar)
|
||||
if model_config.tokenizer_mode == "mistral":
|
||||
_raise_or_fallback(feature_name="--tokenizer-mode mistral",
|
||||
recommend_to_remove=False)
|
||||
return False
|
||||
|
||||
# No CPU offloading yet.
|
||||
if self.cpu_offload_gb != EngineArgs.cpu_offload_gb:
|
||||
_raise_or_fallback(feature_name="--cpu-offload-gb",
|
||||
recommend_to_remove=False)
|
||||
return False
|
||||
|
||||
# Only Fp16 and Bf16 dtypes since we only support FA.
|
||||
V1_SUPPORTED_DTYPES = [torch.bfloat16, torch.float16]
|
||||
if model_config.dtype not in V1_SUPPORTED_DTYPES:
|
||||
_raise_or_fallback(feature_name=f"--dtype {model_config.dtype}",
|
||||
recommend_to_remove=False)
|
||||
return False
|
||||
|
||||
# Some quantization is not compatible with torch.compile.
|
||||
V1_UNSUPPORTED_QUANT = ["bitsandbytes", "gguf"]
|
||||
if model_config.quantization in V1_UNSUPPORTED_QUANT:
|
||||
_raise_or_fallback(
|
||||
feature_name=f"--quantization {model_config.quantization}",
|
||||
recommend_to_remove=False)
|
||||
return False
|
||||
|
||||
# No Embedding Models so far.
|
||||
if model_config.task not in ["generate"]:
|
||||
_raise_or_fallback(feature_name=f"--task {model_config.task}",
|
||||
recommend_to_remove=False)
|
||||
return False
|
||||
|
||||
# No Mamba or Encoder-Decoder so far.
|
||||
if not model_config.is_v1_compatible:
|
||||
_raise_or_fallback(feature_name=model_config.architectures,
|
||||
recommend_to_remove=False)
|
||||
return False
|
||||
|
||||
# No TransformersModel support so far.
|
||||
if (model_config.model_impl == ModelImpl.TRANSFORMERS
|
||||
or model_config.model_impl == "transformers"):
|
||||
_raise_or_fallback(
|
||||
feature_name=f"model_impl={model_config.model_impl}",
|
||||
recommend_to_remove=False)
|
||||
return False
|
||||
|
||||
# No Concurrent Partial Prefills so far.
|
||||
if (self.max_num_partial_prefills
|
||||
!= EngineArgs.max_num_partial_prefills
|
||||
or self.max_long_partial_prefills
|
||||
!= EngineArgs.max_long_partial_prefills
|
||||
or self.long_prefill_token_threshold
|
||||
!= EngineArgs.long_prefill_token_threshold):
|
||||
_raise_or_fallback(feature_name="Concurrent Partial Prefill",
|
||||
recommend_to_remove=False)
|
||||
return False
|
||||
|
||||
# No OTLP observability so far.
|
||||
if (self.otlp_traces_endpoint or self.collect_detailed_traces):
|
||||
_raise_or_fallback(feature_name="--otlp-traces-endpoint",
|
||||
recommend_to_remove=False)
|
||||
return False
|
||||
|
||||
# Only Ngram speculative decoding so far.
|
||||
if (self.speculative_model is not None
|
||||
or self.num_speculative_tokens is not None):
|
||||
# This is supported but experimental (handled below).
|
||||
if self.speculative_model == "[ngram]":
|
||||
pass
|
||||
else:
|
||||
_raise_or_fallback(feature_name="Speculative Decoding",
|
||||
recommend_to_remove=False)
|
||||
return False
|
||||
|
||||
# No Disaggregated Prefill so far.
|
||||
if self.kv_transfer_config != EngineArgs.kv_transfer_config:
|
||||
_raise_or_fallback(feature_name="--kv-transfer-config",
|
||||
recommend_to_remove=False)
|
||||
return False
|
||||
|
||||
# No FlashInfer or XFormers so far.
|
||||
V1_BACKENDS = [
|
||||
"FLASH_ATTN_VLLM_V1", "FLASH_ATTN", "PALLAS", "PALLAS_VLLM_V1",
|
||||
"TRITON_MLA", "FLASHMLA"
|
||||
]
|
||||
if (envs.is_set("VLLM_ATTENTION_BACKEND")
|
||||
and envs.VLLM_ATTENTION_BACKEND not in V1_BACKENDS):
|
||||
name = f"VLLM_ATTENTION_BACKEND={envs.VLLM_ATTENTION_BACKEND}"
|
||||
_raise_or_fallback(feature_name=name, recommend_to_remove=True)
|
||||
return False
|
||||
|
||||
#############################################################
|
||||
# Experimental Features - allow users to opt in.
|
||||
|
||||
# MLA is is supported on V1, but off by default for now.
|
||||
if model_config.use_mla and _warn_or_fallback("MLA"):
|
||||
return False
|
||||
|
||||
# LoRA is supported on V1, but off by default for now.
|
||||
if self.enable_lora and _warn_or_fallback("LORA"):
|
||||
return False
|
||||
|
||||
# PP is supported on V1, but off by default for now.
|
||||
if self.pipeline_parallel_size > 1 and _warn_or_fallback("PP"):
|
||||
return False
|
||||
|
||||
# ngram is supported on V1, but off by default for now.
|
||||
if self.speculative_model == "[ngram]" and _warn_or_fallback("ngram"):
|
||||
return False
|
||||
|
||||
# Non-CUDA is supported on V1, but off by default for now.
|
||||
not_cuda = not current_platform.is_cuda()
|
||||
if not_cuda and _warn_or_fallback( # noqa: SIM103
|
||||
current_platform.device_type):
|
||||
return False
|
||||
#############################################################
|
||||
|
||||
return True
|
||||
|
||||
def _set_default_args_v0(self, model_config: ModelConfig) -> None:
|
||||
"""Set Default Arguments for V0 Engine."""
|
||||
|
||||
max_model_len = model_config.max_model_len
|
||||
use_long_context = max_model_len > 32768
|
||||
if self.enable_chunked_prefill is None:
|
||||
# Chunked prefill not supported for Multimodal or MLA in V0.
|
||||
if model_config.is_multimodal_model or model_config.use_mla:
|
||||
self.enable_chunked_prefill = False
|
||||
|
||||
# Enable chunked prefill by default for long context (> 32K)
|
||||
# models to avoid OOM errors in initial memory profiling phase.
|
||||
elif use_long_context:
|
||||
from vllm.platforms import current_platform
|
||||
is_gpu = current_platform.is_cuda()
|
||||
use_sliding_window = (model_config.get_sliding_window()
|
||||
is not None)
|
||||
use_spec_decode = self.speculative_model is not None
|
||||
|
||||
if (is_gpu and not use_sliding_window and not use_spec_decode
|
||||
and not self.enable_lora
|
||||
and not self.enable_prompt_adapter
|
||||
and model_config.runner_type != "pooling"):
|
||||
self.enable_chunked_prefill = True
|
||||
logger.warning(
|
||||
"Chunked prefill is enabled by default for models "
|
||||
"with max_model_len > 32K. Chunked prefill might "
|
||||
"not work with some features or models. If you "
|
||||
"encounter any issues, please disable by launching "
|
||||
"with --enable-chunked-prefill=False.")
|
||||
|
||||
if self.enable_chunked_prefill is None:
|
||||
self.enable_chunked_prefill = False
|
||||
|
||||
if not self.enable_chunked_prefill and use_long_context:
|
||||
logger.warning(
|
||||
"The model has a long context length (%s). This may cause"
|
||||
"OOM during the initial memory profiling phase, or result "
|
||||
"in low performance due to small KV cache size. Consider "
|
||||
"setting --max-model-len to a smaller value.", max_model_len)
|
||||
elif (self.enable_chunked_prefill
|
||||
and model_config.runner_type == "pooling"):
|
||||
msg = "Chunked prefill is not supported for pooling models"
|
||||
raise ValueError(msg)
|
||||
|
||||
# Disable prefix caching for multimodal models for VLLM_V0.
|
||||
if (model_config.is_multimodal_model and self.enable_prefix_caching):
|
||||
logger.warning(
|
||||
"--enable-prefix-caching is not supported for multimodal "
|
||||
"models in V0 and has been disabled.")
|
||||
self.enable_prefix_caching = False
|
||||
|
||||
# Set max_num_seqs to 256 for VLLM_V0.
|
||||
if self.max_num_seqs is None:
|
||||
self.max_num_seqs = 256
|
||||
|
||||
def _set_default_args_v1(self, usage_context: UsageContext) -> None:
|
||||
"""Set Default Arguments for V1 Engine."""
|
||||
|
||||
# V1 always uses chunked prefills.
|
||||
self.enable_chunked_prefill = True
|
||||
|
||||
# V1 enables prefix caching by default.
|
||||
if self.enable_prefix_caching is None:
|
||||
self.enable_prefix_caching = True
|
||||
|
||||
# V1 should use the new scheduler by default.
|
||||
# Swap it only if this arg is set to the original V0 default
|
||||
if self.scheduler_cls == EngineArgs.scheduler_cls:
|
||||
@@ -1471,19 +1708,21 @@ class EngineArgs:
|
||||
UsageContext.OPENAI_API_SERVER: 2048,
|
||||
}
|
||||
|
||||
use_context_value = usage_context.value if usage_context else None
|
||||
if (self.max_num_batched_tokens is None
|
||||
and usage_context in default_max_num_batched_tokens):
|
||||
self.max_num_batched_tokens = default_max_num_batched_tokens[
|
||||
usage_context]
|
||||
logger.warning(
|
||||
logger.debug(
|
||||
"Setting max_num_batched_tokens to %d for %s usage context.",
|
||||
self.max_num_batched_tokens, usage_context.value)
|
||||
self.max_num_batched_tokens, use_context_value)
|
||||
|
||||
def _override_v1_engine_config(self, engine_config: VllmConfig) -> None:
|
||||
"""
|
||||
Override the EngineConfig's configs based on the usage context for V1.
|
||||
"""
|
||||
assert envs.VLLM_USE_V1, "V1 is not enabled"
|
||||
default_max_num_seqs = 1024
|
||||
if self.max_num_seqs is None:
|
||||
self.max_num_seqs = default_max_num_seqs
|
||||
|
||||
logger.debug("Setting max_num_seqs to %d for %s usage context.",
|
||||
self.max_num_seqs, use_context_value)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -1508,6 +1747,33 @@ class AsyncEngineArgs(EngineArgs):
|
||||
return parser
|
||||
|
||||
|
||||
def _raise_or_fallback(feature_name: str, recommend_to_remove: bool):
|
||||
if envs.is_set("VLLM_USE_V1") and envs.VLLM_USE_V1:
|
||||
raise NotImplementedError(
|
||||
f"VLLM_USE_V1=1 is not supported with {feature_name}.")
|
||||
msg = f"{feature_name} is not supported by the V1 Engine. "
|
||||
msg += "Falling back to V0. "
|
||||
if recommend_to_remove:
|
||||
msg += f"We recommend to remove {feature_name} from your config "
|
||||
msg += "in favor of the V1 Engine."
|
||||
logger.warning(msg)
|
||||
|
||||
|
||||
def _warn_or_fallback(feature_name: str) -> bool:
|
||||
if envs.is_set("VLLM_USE_V1") and envs.VLLM_USE_V1:
|
||||
logger.warning(
|
||||
"Detected VLLM_USE_V1=1 with %s. Usage should "
|
||||
"be considered experimental. Please report any "
|
||||
"issues on Github.", feature_name)
|
||||
should_exit = False
|
||||
else:
|
||||
logger.info(
|
||||
"%s is experimental on VLLM_USE_V1=1. "
|
||||
"Falling back to V0 Engine.", feature_name)
|
||||
should_exit = True
|
||||
return should_exit
|
||||
|
||||
|
||||
# These functions are used by sphinx to build the documentation
|
||||
def _engine_args_parser():
|
||||
return EngineArgs.add_cli_args(FlexibleArgumentParser())
|
||||
|
||||
@@ -595,6 +595,13 @@ class AsyncLLMEngine(EngineClient):
|
||||
log_requests: bool = True,
|
||||
start_engine_loop: bool = True,
|
||||
**kwargs) -> None:
|
||||
if envs.VLLM_USE_V1:
|
||||
raise ValueError(
|
||||
"Using V0 AsyncLLMEngine, but envs.VLLM_USE_V1=True. "
|
||||
"This should not happen. As a workaround, try using "
|
||||
"AsyncLLMEngine.from_vllm_config(...) or explicitly set "
|
||||
"VLLM_USE_V1=0 or 1 and report this issue on Github.")
|
||||
|
||||
self.log_requests = log_requests
|
||||
self.engine = self._engine_class(*args, **kwargs)
|
||||
|
||||
@@ -629,33 +636,53 @@ class AsyncLLMEngine(EngineClient):
|
||||
engine_config: VllmConfig) -> Type[ExecutorBase]:
|
||||
return LLMEngine._get_executor_cls(engine_config)
|
||||
|
||||
@classmethod
|
||||
def from_vllm_config(
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
start_engine_loop: bool = True,
|
||||
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
|
||||
stat_loggers: Optional[dict[str, StatLoggerBase]] = None,
|
||||
disable_log_requests: bool = False,
|
||||
disable_log_stats: bool = False,
|
||||
) -> "AsyncLLMEngine":
|
||||
"""Create an AsyncLLMEngine from the EngineArgs."""
|
||||
|
||||
return cls(
|
||||
vllm_config=vllm_config,
|
||||
executor_class=cls._get_executor_cls(vllm_config),
|
||||
start_engine_loop=start_engine_loop,
|
||||
log_requests=not disable_log_requests,
|
||||
log_stats=not disable_log_stats,
|
||||
usage_context=usage_context,
|
||||
stat_loggers=stat_loggers,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_engine_args(
|
||||
cls,
|
||||
engine_args: AsyncEngineArgs,
|
||||
engine_config: Optional[VllmConfig] = None,
|
||||
start_engine_loop: bool = True,
|
||||
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
|
||||
stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
|
||||
) -> "AsyncLLMEngine":
|
||||
"""Creates an async LLM engine from the engine arguments."""
|
||||
# Create the engine configs.
|
||||
if engine_config is None:
|
||||
engine_config = engine_args.create_engine_config(usage_context)
|
||||
|
||||
executor_class = cls._get_executor_cls(engine_config)
|
||||
vllm_config = engine_args.create_engine_config(usage_context)
|
||||
|
||||
# Create the async LLM engine.
|
||||
engine = cls(
|
||||
vllm_config=engine_config,
|
||||
executor_class=executor_class,
|
||||
log_requests=not engine_args.disable_log_requests,
|
||||
log_stats=not engine_args.disable_log_stats,
|
||||
async_engine_cls = cls
|
||||
if envs.VLLM_USE_V1:
|
||||
from vllm.v1.engine.async_llm import AsyncLLM as V1AsyncLLMEngine
|
||||
async_engine_cls = V1AsyncLLMEngine
|
||||
|
||||
return async_engine_cls.from_vllm_config(
|
||||
vllm_config=vllm_config,
|
||||
start_engine_loop=start_engine_loop,
|
||||
usage_context=usage_context,
|
||||
stat_loggers=stat_loggers,
|
||||
disable_log_stats=engine_args.disable_log_stats,
|
||||
disable_log_requests=engine_args.disable_log_requests,
|
||||
)
|
||||
return engine
|
||||
|
||||
@property
|
||||
def is_running(self) -> bool:
|
||||
@@ -1203,7 +1230,7 @@ class AsyncLLMEngine(EngineClient):
|
||||
|
||||
|
||||
# TODO(v1): Remove this class proxy when V1 goes default.
|
||||
if envs.VLLM_USE_V1:
|
||||
if envs.is_set("VLLM_USE_V1") and envs.VLLM_USE_V1:
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
AsyncLLMEngine = AsyncLLM # type: ignore
|
||||
|
||||
+35
-12
@@ -216,6 +216,12 @@ class LLMEngine:
|
||||
mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
|
||||
use_cached_outputs: bool = False,
|
||||
) -> None:
|
||||
if envs.VLLM_USE_V1:
|
||||
raise ValueError(
|
||||
"Using V0 LLMEngine, but envs.VLLM_USE_V1=True. "
|
||||
"This should not happen. As a workaround, try using "
|
||||
"LLMEngine.from_vllm_config(...) or explicitly set "
|
||||
"VLLM_USE_V1=0 or 1 and report this issue on Github.")
|
||||
|
||||
self.vllm_config = vllm_config
|
||||
self.model_config = vllm_config.model_config
|
||||
@@ -479,6 +485,22 @@ class LLMEngine:
|
||||
f"{distributed_executor_backend}")
|
||||
return executor_class
|
||||
|
||||
@classmethod
|
||||
def from_vllm_config(
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
|
||||
stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
|
||||
disable_log_stats: bool = False,
|
||||
) -> "LLMEngine":
|
||||
return cls(
|
||||
vllm_config=vllm_config,
|
||||
executor_class=cls._get_executor_cls(vllm_config),
|
||||
log_stats=(not disable_log_stats),
|
||||
usage_context=usage_context,
|
||||
stat_loggers=stat_loggers,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_engine_args(
|
||||
cls,
|
||||
@@ -488,19 +510,20 @@ class LLMEngine:
|
||||
) -> "LLMEngine":
|
||||
"""Creates an LLM engine from the engine arguments."""
|
||||
# Create the engine configs.
|
||||
engine_config = engine_args.create_engine_config(usage_context)
|
||||
executor_class = cls._get_executor_cls(engine_config)
|
||||
# Create the LLM engine.
|
||||
engine = cls(
|
||||
vllm_config=engine_config,
|
||||
executor_class=executor_class,
|
||||
log_stats=not engine_args.disable_log_stats,
|
||||
vllm_config = engine_args.create_engine_config(usage_context)
|
||||
|
||||
engine_cls = cls
|
||||
if envs.VLLM_USE_V1:
|
||||
from vllm.v1.engine.llm_engine import LLMEngine as V1LLMEngine
|
||||
engine_cls = V1LLMEngine
|
||||
|
||||
return engine_cls.from_vllm_config(
|
||||
vllm_config=vllm_config,
|
||||
usage_context=usage_context,
|
||||
stat_loggers=stat_loggers,
|
||||
disable_log_stats=engine_args.disable_log_stats,
|
||||
)
|
||||
|
||||
return engine
|
||||
|
||||
def __reduce__(self):
|
||||
# This is to ensure that the LLMEngine is not referenced in
|
||||
# the closure used to initialize Ray worker actors
|
||||
@@ -2097,6 +2120,6 @@ class LLMEngine:
|
||||
return sampling_params
|
||||
|
||||
|
||||
# TODO(v1): Remove this class proxy when V1 goes default.
|
||||
if envs.VLLM_USE_V1:
|
||||
from vllm.v1.engine.llm_engine import LLMEngine # type: ignore
|
||||
if envs.is_set("VLLM_USE_V1") and envs.VLLM_USE_V1:
|
||||
from vllm.v1.engine.llm_engine import LLMEngine as V1LLMEngine
|
||||
LLMEngine = V1LLMEngine # type: ignore
|
||||
|
||||
@@ -18,7 +18,6 @@ from zmq.asyncio import Socket
|
||||
from vllm import PoolingParams
|
||||
from vllm.config import DecodingConfig, ModelConfig, VllmConfig
|
||||
from vllm.core.scheduler import SchedulerOutputs
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
# yapf conflicts with isort for this block
|
||||
# yapf: disable
|
||||
from vllm.engine.async_llm_engine import (
|
||||
@@ -133,9 +132,9 @@ class MQLLMEngineClient(EngineClient):
|
||||
self._engine_process = psutil.Process(engine_pid)
|
||||
|
||||
@staticmethod
|
||||
def is_unsupported_config(engine_args: AsyncEngineArgs):
|
||||
def is_unsupported_config(vllm_config: VllmConfig):
|
||||
# Pipeline parallel not yet supported
|
||||
return engine_args.pipeline_parallel_size > 1
|
||||
return vllm_config.parallel_config.pipeline_parallel_size > 1
|
||||
|
||||
@contextmanager
|
||||
def get_data_socket(self) -> Iterator[Socket]:
|
||||
|
||||
@@ -9,6 +9,7 @@ import cloudpickle
|
||||
import zmq
|
||||
|
||||
from vllm import AsyncEngineArgs, SamplingParams
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.engine.llm_engine import LLMEngine
|
||||
# yapf conflicts with isort for this block
|
||||
# yapf: disable
|
||||
@@ -110,25 +111,39 @@ class MQLLMEngine:
|
||||
return ENGINE_DEAD_ERROR()
|
||||
|
||||
@classmethod
|
||||
def from_engine_args(cls, engine_args: AsyncEngineArgs,
|
||||
usage_context: UsageContext, ipc_path: str):
|
||||
"""Creates an MQLLMEngine from the engine arguments."""
|
||||
def from_vllm_config(cls, vllm_config: VllmConfig,
|
||||
usage_context: UsageContext,
|
||||
disable_log_requests: bool, disable_log_stats: bool,
|
||||
ipc_path: str) -> "MQLLMEngine":
|
||||
# Setup plugins for each process
|
||||
from vllm.plugins import load_general_plugins
|
||||
load_general_plugins()
|
||||
|
||||
engine_config = engine_args.create_engine_config(usage_context)
|
||||
executor_class = LLMEngine._get_executor_cls(engine_config)
|
||||
use_async_sockets = vllm_config.model_config.use_async_output_proc
|
||||
|
||||
use_async_sockets = engine_config.model_config.use_async_output_proc
|
||||
return cls(
|
||||
vllm_config=vllm_config,
|
||||
executor_class=LLMEngine._get_executor_cls(vllm_config),
|
||||
ipc_path=ipc_path,
|
||||
usage_context=usage_context,
|
||||
use_async_sockets=use_async_sockets,
|
||||
log_requests=(not disable_log_requests),
|
||||
log_stats=(not disable_log_stats),
|
||||
)
|
||||
|
||||
return cls(ipc_path=ipc_path,
|
||||
use_async_sockets=use_async_sockets,
|
||||
vllm_config=engine_config,
|
||||
executor_class=executor_class,
|
||||
log_requests=not engine_args.disable_log_requests,
|
||||
log_stats=not engine_args.disable_log_stats,
|
||||
usage_context=usage_context)
|
||||
@staticmethod
|
||||
def from_engine_args(engine_args: AsyncEngineArgs,
|
||||
usage_context: UsageContext, ipc_path: str):
|
||||
"""Creates an MQLLMEngine from the engine arguments."""
|
||||
|
||||
vllm_config = engine_args.create_engine_config(usage_context)
|
||||
return MQLLMEngine.from_vllm_config(
|
||||
ipc_path=ipc_path,
|
||||
vllm_config=vllm_config,
|
||||
usage_context=usage_context,
|
||||
disable_log_requests=engine_args.disable_log_requests,
|
||||
disable_log_stats=engine_args.disable_log_stats,
|
||||
)
|
||||
|
||||
def start(self):
|
||||
try:
|
||||
@@ -396,12 +411,16 @@ def signal_handler(*_) -> None:
|
||||
raise KeyboardInterrupt("MQLLMEngine terminated")
|
||||
|
||||
|
||||
def run_mp_engine(engine_args: AsyncEngineArgs, usage_context: UsageContext,
|
||||
ipc_path: str, engine_alive):
|
||||
def run_mp_engine(vllm_config: VllmConfig, usage_context: UsageContext,
|
||||
ipc_path: str, disable_log_stats: bool,
|
||||
disable_log_requests: bool, engine_alive):
|
||||
try:
|
||||
engine = MQLLMEngine.from_engine_args(engine_args=engine_args,
|
||||
usage_context=usage_context,
|
||||
ipc_path=ipc_path)
|
||||
engine = MQLLMEngine.from_vllm_config(
|
||||
vllm_config=vllm_config,
|
||||
usage_context=usage_context,
|
||||
disable_log_stats=disable_log_stats,
|
||||
disable_log_requests=disable_log_requests,
|
||||
ipc_path=ipc_path)
|
||||
|
||||
signal.signal(signal.SIGTERM, signal_handler)
|
||||
|
||||
|
||||
+5
-14
@@ -11,7 +11,6 @@ import torch.nn as nn
|
||||
from tqdm import tqdm
|
||||
from typing_extensions import TypeVar, deprecated
|
||||
|
||||
from vllm import envs
|
||||
from vllm.beam_search import (BeamSearchInstance, BeamSearchOutput,
|
||||
BeamSearchSequence, get_beam_search_score)
|
||||
from vllm.config import CompilationConfig
|
||||
@@ -238,23 +237,15 @@ class LLM:
|
||||
compilation_config=compilation_config_instance,
|
||||
**kwargs,
|
||||
)
|
||||
# Logic to switch between engines is done at runtime instead of import
|
||||
# to avoid import order issues
|
||||
self.engine_class = self.get_engine_class()
|
||||
self.llm_engine = self.engine_class.from_engine_args(
|
||||
engine_args, usage_context=UsageContext.LLM_CLASS)
|
||||
|
||||
# Create the Engine (autoselects V0 vs V1)
|
||||
self.llm_engine = LLMEngine.from_engine_args(
|
||||
engine_args=engine_args, usage_context=UsageContext.LLM_CLASS)
|
||||
self.engine_class = type(self.llm_engine)
|
||||
|
||||
self.request_counter = Counter()
|
||||
self.default_sampling_params: Union[dict[str, Any], None] = None
|
||||
|
||||
@staticmethod
|
||||
def get_engine_class() -> type[LLMEngine]:
|
||||
if envs.VLLM_USE_V1:
|
||||
# Lazy import: the v1 package isn't distributed
|
||||
from vllm.v1.engine.llm_engine import LLMEngine as V1LLMEngine
|
||||
return V1LLMEngine # type: ignore
|
||||
return LLMEngine
|
||||
|
||||
def get_tokenizer(self) -> AnyTokenizer:
|
||||
return self.llm_engine.get_tokenizer_group(TokenizerGroup).tokenizer
|
||||
|
||||
|
||||
@@ -154,21 +154,47 @@ async def build_async_engine_client_from_engine_args(
|
||||
Returns the Client or None if the creation failed.
|
||||
"""
|
||||
|
||||
# AsyncLLMEngine.
|
||||
if (MQLLMEngineClient.is_unsupported_config(engine_args)
|
||||
or envs.VLLM_USE_V1 or disable_frontend_multiprocessing):
|
||||
# Create the EngineConfig (determines if we can use V1).
|
||||
usage_context = UsageContext.OPENAI_API_SERVER
|
||||
vllm_config = engine_args.create_engine_config(usage_context=usage_context)
|
||||
|
||||
# V1 AsyncLLM.
|
||||
if envs.VLLM_USE_V1:
|
||||
if disable_frontend_multiprocessing:
|
||||
logger.warning(
|
||||
"V1 is enabled, but got --disable-frontend-multiprocessing. "
|
||||
"To disable frontend multiprocessing, set VLLM_USE_V1=0.")
|
||||
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
async_llm: Optional[AsyncLLM] = None
|
||||
try:
|
||||
async_llm = AsyncLLM.from_vllm_config(
|
||||
vllm_config=vllm_config,
|
||||
usage_context=usage_context,
|
||||
disable_log_requests=engine_args.disable_log_requests,
|
||||
disable_log_stats=engine_args.disable_log_stats)
|
||||
yield async_llm
|
||||
finally:
|
||||
if async_llm:
|
||||
async_llm.shutdown()
|
||||
|
||||
# V0 AsyncLLM.
|
||||
elif (MQLLMEngineClient.is_unsupported_config(vllm_config)
|
||||
or disable_frontend_multiprocessing):
|
||||
|
||||
engine_client: Optional[EngineClient] = None
|
||||
try:
|
||||
engine_client = AsyncLLMEngine.from_engine_args(
|
||||
engine_args=engine_args,
|
||||
usage_context=UsageContext.OPENAI_API_SERVER)
|
||||
engine_client = AsyncLLMEngine.from_vllm_config(
|
||||
vllm_config=vllm_config,
|
||||
usage_context=usage_context,
|
||||
disable_log_requests=engine_args.disable_log_requests,
|
||||
disable_log_stats=engine_args.disable_log_stats)
|
||||
yield engine_client
|
||||
finally:
|
||||
if engine_client and hasattr(engine_client, "shutdown"):
|
||||
engine_client.shutdown()
|
||||
|
||||
# MQLLMEngine.
|
||||
# V0MQLLMEngine.
|
||||
else:
|
||||
if "PROMETHEUS_MULTIPROC_DIR" not in os.environ:
|
||||
# Make TemporaryDirectory for prometheus multiprocessing
|
||||
@@ -199,10 +225,11 @@ async def build_async_engine_client_from_engine_args(
|
||||
# not actually result in an exitcode being reported. As a result
|
||||
# we use a shared variable to communicate the information.
|
||||
engine_alive = multiprocessing.Value('b', True, lock=False)
|
||||
engine_process = context.Process(target=run_mp_engine,
|
||||
args=(engine_args,
|
||||
UsageContext.OPENAI_API_SERVER,
|
||||
ipc_path, engine_alive))
|
||||
engine_process = context.Process(
|
||||
target=run_mp_engine,
|
||||
args=(vllm_config, UsageContext.OPENAI_API_SERVER, ipc_path,
|
||||
engine_args.disable_log_stats,
|
||||
engine_args.disable_log_requests, engine_alive))
|
||||
engine_process.start()
|
||||
engine_pid = engine_process.pid
|
||||
assert engine_pid is not None, "Engine process failed to start."
|
||||
@@ -217,8 +244,7 @@ async def build_async_engine_client_from_engine_args(
|
||||
atexit.register(_cleanup_ipc_path)
|
||||
|
||||
# Build RPCClient, which conforms to EngineClient Protocol.
|
||||
engine_config = engine_args.create_engine_config()
|
||||
build_client = partial(MQLLMEngineClient, ipc_path, engine_config,
|
||||
build_client = partial(MQLLMEngineClient, ipc_path, vllm_config,
|
||||
engine_pid)
|
||||
mq_engine_client = await asyncio.get_running_loop().run_in_executor(
|
||||
None, build_client)
|
||||
|
||||
+18
-2
@@ -74,7 +74,7 @@ if TYPE_CHECKING:
|
||||
VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False
|
||||
VLLM_SKIP_P2P_CHECK: bool = False
|
||||
VLLM_DISABLED_KERNELS: list[str] = []
|
||||
VLLM_USE_V1: bool = False
|
||||
VLLM_USE_V1: bool = True
|
||||
VLLM_ROCM_FP8_PADDING: bool = True
|
||||
VLLM_ENABLE_V1_MULTIPROCESSING: bool = True
|
||||
VLLM_LOG_BATCHSIZE_INTERVAL: float = -1
|
||||
@@ -522,7 +522,7 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
|
||||
# If set, use the V1 code path.
|
||||
"VLLM_USE_V1":
|
||||
lambda: bool(int(os.getenv("VLLM_USE_V1", "0"))),
|
||||
lambda: bool(int(os.getenv("VLLM_USE_V1", "1"))),
|
||||
|
||||
# Pad the fp8 weights to 256 bytes for ROCm
|
||||
"VLLM_ROCM_FP8_PADDING":
|
||||
@@ -644,3 +644,19 @@ def __getattr__(name: str):
|
||||
|
||||
def __dir__():
|
||||
return list(environment_variables.keys())
|
||||
|
||||
|
||||
def is_set(name: str):
|
||||
"""Check if an environment variable is explicitly set."""
|
||||
if name in environment_variables:
|
||||
return name in os.environ
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
|
||||
def set_vllm_use_v1(use_v1: bool):
|
||||
if is_set("VLLM_USE_V1"):
|
||||
raise ValueError(
|
||||
"Should not call set_vllm_use_v1() if VLLM_USE_V1 is set "
|
||||
"explicitly by the user. Please raise this as a Github "
|
||||
"Issue and explicitly set VLLM_USE_V1=0 or 1.")
|
||||
os.environ["VLLM_USE_V1"] = "1" if use_v1 else "0"
|
||||
|
||||
@@ -74,7 +74,8 @@ def resolve_transformers_fallback(model_config: ModelConfig,
|
||||
if not is_transformers_impl_compatible(arch, custom_model_module):
|
||||
raise ValueError(
|
||||
f"{arch} has no vLLM implementation and the Transformers "
|
||||
"implementation is not compatible with vLLM.")
|
||||
"implementation is not compatible with vLLM. Try setting "
|
||||
"VLLM_USE_V1=0.")
|
||||
logger.warning(
|
||||
"%s has no vLLM implementation, falling back to Transformers "
|
||||
"implementation. Some features may not be supported and "
|
||||
|
||||
@@ -42,7 +42,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.sampling_metadata import SamplingMetadata
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import SupportsPP
|
||||
from .interfaces import SupportsPP, SupportsV0Only
|
||||
from .utils import (is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory, make_layers,
|
||||
maybe_prefix)
|
||||
@@ -279,7 +279,7 @@ class BloomModel(nn.Module):
|
||||
return hidden_states
|
||||
|
||||
|
||||
class BloomForCausalLM(nn.Module, SupportsPP):
|
||||
class BloomForCausalLM(nn.Module, SupportsPP, SupportsV0Only):
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
@@ -3,10 +3,11 @@
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.models.llama import LlamaForCausalLM
|
||||
|
||||
from .interfaces import SupportsV0Only
|
||||
from .utils import PPMissingLayer
|
||||
|
||||
|
||||
class GlmForCausalLM(LlamaForCausalLM):
|
||||
class GlmForCausalLM(LlamaForCausalLM, SupportsV0Only):
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
|
||||
@@ -36,7 +36,7 @@ from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.ultravox import UltravoxConfig
|
||||
|
||||
from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
|
||||
SupportsMultiModal, SupportsPP)
|
||||
SupportsMultiModal, SupportsPP, SupportsV0Only)
|
||||
from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
|
||||
init_vllm_registered_model, maybe_prefix,
|
||||
merge_multimodal_embeddings,
|
||||
@@ -405,7 +405,8 @@ class ModifiedWhisperEncoder(WhisperEncoder):
|
||||
UltravoxMultiModalProcessor,
|
||||
info=UltravoxProcessingInfo,
|
||||
dummy_inputs=UltravoxDummyInputsBuilder)
|
||||
class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
|
||||
class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,
|
||||
SupportsV0Only):
|
||||
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||
|
||||
@@ -196,7 +196,8 @@ class FlashAttentionImpl(AttentionImpl):
|
||||
if head_size not in support_head_sizes:
|
||||
raise ValueError(
|
||||
f"Head size {head_size} is not supported by FlashAttention. "
|
||||
f"Supported head sizes are: {support_head_sizes}.")
|
||||
f"Supported head sizes are: {support_head_sizes}. "
|
||||
"Set VLLM_USE_V1=0 to use another attention backend.")
|
||||
|
||||
if attn_type != AttentionType.DECODER:
|
||||
raise NotImplementedError("Encoder self-attention and "
|
||||
|
||||
@@ -8,6 +8,7 @@ from typing import Optional, Union
|
||||
|
||||
import numpy as np
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.engine.protocol import EngineClient
|
||||
@@ -49,6 +50,12 @@ class AsyncLLM(EngineClient):
|
||||
log_requests: bool = True,
|
||||
start_engine_loop: bool = True,
|
||||
) -> None:
|
||||
if not envs.VLLM_USE_V1:
|
||||
raise ValueError(
|
||||
"Using V1 AsyncLLMEngine, but envs.VLLM_USE_V1=False. "
|
||||
"This should not happen. As a workaround, try using "
|
||||
"AsyncLLMEngine.from_vllm_config(...) or explicitly set "
|
||||
"VLLM_USE_V1=0 or 1 and report this issue on Github.")
|
||||
|
||||
assert start_engine_loop
|
||||
|
||||
@@ -92,22 +99,50 @@ class AsyncLLM(EngineClient):
|
||||
|
||||
self.output_handler: Optional[asyncio.Task] = None
|
||||
|
||||
@classmethod
|
||||
def from_vllm_config(
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
start_engine_loop: bool = True,
|
||||
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
|
||||
stat_loggers: Optional[dict[str, StatLoggerBase]] = None,
|
||||
disable_log_requests: bool = False,
|
||||
disable_log_stats: bool = False,
|
||||
) -> "AsyncLLM":
|
||||
if not envs.VLLM_USE_V1:
|
||||
raise ValueError(
|
||||
"Using V1 AsyncLLMEngine, but envs.VLLM_USE_V1=False. "
|
||||
"This should not happen. As a workaround, try using "
|
||||
"AsyncLLMEngine.from_vllm_config(...) or explicitly set "
|
||||
"VLLM_USE_V1=0 or 1 and report this issue on Github.")
|
||||
|
||||
# FIXME(rob): refactor VllmConfig to include the StatLoggers
|
||||
# include StatLogger in the Oracle decision.
|
||||
if stat_loggers is not None:
|
||||
raise ValueError("Custom StatLoggers are not yet supported on V1. "
|
||||
"Explicitly set VLLM_USE_V1=0 to disable V1.")
|
||||
|
||||
# Create the LLMEngine.
|
||||
return cls(
|
||||
vllm_config=vllm_config,
|
||||
executor_class=Executor.get_class(vllm_config),
|
||||
start_engine_loop=start_engine_loop,
|
||||
log_requests=not disable_log_requests,
|
||||
log_stats=not disable_log_stats,
|
||||
usage_context=usage_context,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_engine_args(
|
||||
cls,
|
||||
engine_args: AsyncEngineArgs,
|
||||
engine_config: Optional[VllmConfig] = None,
|
||||
start_engine_loop: bool = True,
|
||||
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
|
||||
) -> "AsyncLLM":
|
||||
"""Create an AsyncLLM from the EngineArgs."""
|
||||
|
||||
# Create the engine configs.
|
||||
if engine_config is None:
|
||||
vllm_config = engine_args.create_engine_config(usage_context)
|
||||
else:
|
||||
vllm_config = engine_config
|
||||
|
||||
vllm_config = engine_args.create_engine_config(usage_context)
|
||||
executor_class = Executor.get_class(vllm_config)
|
||||
|
||||
# Create the AsyncLLM.
|
||||
|
||||
@@ -46,6 +46,13 @@ class LLMEngine:
|
||||
use_cached_outputs: bool = False,
|
||||
multiprocess_mode: bool = False,
|
||||
) -> None:
|
||||
if not envs.VLLM_USE_V1:
|
||||
raise ValueError(
|
||||
"Using V1 LLMEngine, but envs.VLLM_USE_V1=False. "
|
||||
"This should not happen. As a workaround, try using "
|
||||
"LLMEngine.from_vllm_config(...) or explicitly set "
|
||||
"VLLM_USE_V1=0 or 1 and report this issue on Github.")
|
||||
|
||||
self.vllm_config = vllm_config
|
||||
self.model_config = vllm_config.model_config
|
||||
self.cache_config = vllm_config.cache_config
|
||||
@@ -88,6 +95,26 @@ class LLMEngine:
|
||||
# for v0 compatibility
|
||||
self.model_executor = self.engine_core.engine_core.model_executor # type: ignore
|
||||
|
||||
@classmethod
|
||||
def from_vllm_config(
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
|
||||
stat_loggers: Optional[dict[str, StatLoggerBase]] = None,
|
||||
disable_log_stats: bool = False,
|
||||
) -> "LLMEngine":
|
||||
if stat_loggers is not None:
|
||||
raise NotImplementedError(
|
||||
"Passing StatLoggers to V1 is not yet supported. "
|
||||
"Set VLLM_USE_V1=0 and file and issue on Github.")
|
||||
|
||||
return cls(vllm_config=vllm_config,
|
||||
executor_class=Executor.get_class(vllm_config),
|
||||
log_stats=(not disable_log_stats),
|
||||
usage_context=usage_context,
|
||||
stat_loggers=stat_loggers,
|
||||
multiprocess_mode=envs.VLLM_ENABLE_V1_MULTIPROCESSING)
|
||||
|
||||
@classmethod
|
||||
def from_engine_args(
|
||||
cls,
|
||||
|
||||
@@ -184,7 +184,7 @@ class Processor:
|
||||
# Only applicable to multimodal models with legacy input processor.
|
||||
processed_inputs = self.input_processor(preprocessed_inputs)
|
||||
|
||||
self._validate_model_inputs(processed_inputs)
|
||||
self._validate_model_inputs(processed_inputs, lora_request)
|
||||
|
||||
if is_encoder_decoder_inputs(processed_inputs):
|
||||
decoder_inputs = SingletonInputsAdapter(
|
||||
@@ -200,8 +200,12 @@ class Processor:
|
||||
raise NotImplementedError
|
||||
|
||||
assert isinstance(params, SamplingParams)
|
||||
# TODO: can we avoid cloning here in multiproc case
|
||||
# TODO: can we avoid cloning here in multiproc case?
|
||||
sampling_params = params.clone()
|
||||
# If unset max tokens, then generate up to the max_model_len.
|
||||
if sampling_params.max_tokens is None:
|
||||
sampling_params.max_tokens = (self.model_config.max_model_len -
|
||||
len(decoder_inputs.prompt_token_ids))
|
||||
sampling_params.update_from_generation_config(
|
||||
self.generation_config_fields, eos_token_id)
|
||||
sampling_params.update_from_tokenizer(
|
||||
@@ -296,7 +300,9 @@ class Processor:
|
||||
lora_request=lora_request,
|
||||
)
|
||||
|
||||
def _validate_model_inputs(self, inputs: ProcessorInputs):
|
||||
def _validate_model_inputs(self,
|
||||
inputs: ProcessorInputs,
|
||||
lora_request: Optional[LoRARequest] = None):
|
||||
if is_encoder_decoder_inputs(inputs):
|
||||
# For encoder-decoder multimodal models, the max_prompt_len
|
||||
# restricts the decoder prompt length
|
||||
@@ -310,6 +316,13 @@ class Processor:
|
||||
if prompt_ids is None or len(prompt_ids) == 0:
|
||||
raise ValueError("Prompt cannot be empty")
|
||||
|
||||
max_input_id = max(prompt_ids)
|
||||
max_allowed = self.tokenizer.get_lora_tokenizer(
|
||||
lora_request).max_token_id
|
||||
if max_input_id > max_allowed:
|
||||
raise ValueError(
|
||||
"Token id {} is out of vocabulary".format(max_input_id))
|
||||
|
||||
if len(prompt_ids) >= self.model_config.max_model_len:
|
||||
raise ValueError(
|
||||
f"Prompt length of {len(prompt_ids)} is longer than the "
|
||||
|
||||
Reference in New Issue
Block a user