[Bugfix][V1] Read ModelState max_model_len from model config (#58149)
Signed-off-by: Chenglun Hu <chenglunhu@gmail.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -21,7 +21,7 @@ from vllm.v1.worker.gpu.model_states.recoverssm import RecoverSSMState
|
|||||||
def test_prepare_attn_forwards_positions(monkeypatch: pytest.MonkeyPatch) -> None:
|
def test_prepare_attn_forwards_positions(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
state = object.__new__(MambaHybridModelState)
|
state = object.__new__(MambaHybridModelState)
|
||||||
state.vllm_config = SimpleNamespace(num_speculative_tokens=0)
|
state.vllm_config = SimpleNamespace(num_speculative_tokens=0)
|
||||||
state.max_model_len = 8192
|
state.model_config = SimpleNamespace(max_model_len=8192)
|
||||||
state._align_mode = False
|
state._align_mode = False
|
||||||
state.recoverssm = None
|
state.recoverssm = None
|
||||||
|
|
||||||
|
|||||||
@@ -59,7 +59,6 @@ class ModelState(ABC):
|
|||||||
self.model = model
|
self.model = model
|
||||||
self.device = device
|
self.device = device
|
||||||
|
|
||||||
self.max_model_len = self.model_config.max_model_len
|
|
||||||
self.max_num_reqs = self.scheduler_config.max_num_seqs
|
self.max_num_reqs = self.scheduler_config.max_num_seqs
|
||||||
self.max_num_tokens = self.scheduler_config.max_num_batched_tokens
|
self.max_num_tokens = self.scheduler_config.max_num_batched_tokens
|
||||||
self.inputs_embeds_size = self.model_config.get_inputs_embeds_size()
|
self.inputs_embeds_size = self.model_config.get_inputs_embeds_size()
|
||||||
@@ -99,6 +98,11 @@ class ModelState(ABC):
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def max_model_len(self) -> int:
|
||||||
|
# Auto-fit can reduce the limit after model state initialization.
|
||||||
|
return self.model_config.max_model_len
|
||||||
|
|
||||||
def get_supported_generation_tasks(self) -> tuple[GenerationTask, ...]:
|
def get_supported_generation_tasks(self) -> tuple[GenerationTask, ...]:
|
||||||
from vllm.model_executor.models.interfaces import (
|
from vllm.model_executor.models.interfaces import (
|
||||||
supports_realtime,
|
supports_realtime,
|
||||||
|
|||||||
Reference in New Issue
Block a user