[Speculators] Move tests + fix integration (#27308)
Signed-off-by: Dipika Sikka <dipikasikka1@gmail.com> Signed-off-by: Rahul Tuli <rtuli@redhat.com> Signed-off-by: rahul-tuli <rtuli@redhat.com> Co-authored-by: Rahul Tuli <rtuli@redhat.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
co-authored by
Rahul Tuli
Robert Shaw
parent
8b62495076
commit
413ef7a3b4
+17
-11
@@ -81,7 +81,7 @@ from vllm.transformers_utils.config import (
|
||||
is_interleaved,
|
||||
maybe_override_with_speculators,
|
||||
)
|
||||
from vllm.transformers_utils.utils import check_gguf_file
|
||||
from vllm.transformers_utils.utils import check_gguf_file, is_s3
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
from vllm.utils.network_utils import get_ip
|
||||
@@ -1305,20 +1305,26 @@ class EngineArgs:
|
||||
|
||||
device_config = DeviceConfig(device=cast(Device, current_platform.device_type))
|
||||
|
||||
# Check if the model is a speculator and override model/tokenizer/config
|
||||
# BEFORE creating ModelConfig, so the config is created with the target model
|
||||
# Skip speculator detection for S3 models since HuggingFace cannot load
|
||||
# configs directly from S3 URLs. S3 models can still use speculators with
|
||||
# explicit --speculative-config.
|
||||
if not is_s3(self.model):
|
||||
(self.model, self.tokenizer, self.speculative_config) = (
|
||||
maybe_override_with_speculators(
|
||||
model=self.model,
|
||||
tokenizer=self.tokenizer,
|
||||
revision=self.revision,
|
||||
trust_remote_code=self.trust_remote_code,
|
||||
vllm_speculative_config=self.speculative_config,
|
||||
)
|
||||
)
|
||||
|
||||
model_config = self.create_model_config()
|
||||
self.model = model_config.model
|
||||
self.tokenizer = model_config.tokenizer
|
||||
|
||||
(self.model, self.tokenizer, self.speculative_config) = (
|
||||
maybe_override_with_speculators(
|
||||
model=self.model,
|
||||
tokenizer=self.tokenizer,
|
||||
revision=self.revision,
|
||||
trust_remote_code=self.trust_remote_code,
|
||||
vllm_speculative_config=self.speculative_config,
|
||||
)
|
||||
)
|
||||
|
||||
# * If VLLM_USE_V1 is unset, we enable V1 for "supported features"
|
||||
# and fall back to V0 for experimental or unsupported features.
|
||||
# * If VLLM_USE_V1=1, we enable V1 for supported + experimental
|
||||
|
||||
Reference in New Issue
Block a user