[Model] Support Mamba (#6484)
This commit is contained in:
@@ -36,10 +36,10 @@ class BlockSpaceManager(ABC):
|
||||
from vllm.core.block_manager_v2 import BlockSpaceManagerV2
|
||||
return BlockSpaceManagerV2
|
||||
|
||||
if version == "embedding":
|
||||
from vllm.core.embedding_model_block_manager import (
|
||||
EmbeddingModelBlockSpaceManager)
|
||||
return EmbeddingModelBlockSpaceManager
|
||||
if version == "placeholder":
|
||||
from vllm.core.placeholder_block_space_manager import (
|
||||
PlaceholderBlockSpaceManager)
|
||||
return PlaceholderBlockSpaceManager
|
||||
|
||||
raise ValueError(f"Unknown version {version=}")
|
||||
|
||||
|
||||
+5
-4
@@ -5,9 +5,10 @@ from vllm.sequence import Sequence, SequenceGroup
|
||||
from vllm.utils import Device
|
||||
|
||||
|
||||
class EmbeddingModelBlockSpaceManager(BlockSpaceManager):
|
||||
"""An embedding version of BlockSpaceManager for use in environments
|
||||
with embedding models where block management is not required.
|
||||
class PlaceholderBlockSpaceManager(BlockSpaceManager):
|
||||
"""A version of BlockSpaceManager for use in environments
|
||||
where block management is not required.
|
||||
For example: embedding models or attention-free models like Mamba.
|
||||
|
||||
This class provides the same interface as BlockSpaceManager, but its
|
||||
methods perform no actions or return simple values like True in specific
|
||||
@@ -40,7 +41,7 @@ class EmbeddingModelBlockSpaceManager(BlockSpaceManager):
|
||||
seq: Sequence,
|
||||
num_lookahead_slots: int,
|
||||
) -> List[Tuple[int, int]]:
|
||||
return None # type: ignore
|
||||
return []
|
||||
|
||||
def fork(self, parent_seq: Sequence, child_seq: Sequence) -> None:
|
||||
pass
|
||||
@@ -314,8 +314,9 @@ class Scheduler:
|
||||
version = "v1"
|
||||
if self.scheduler_config.use_v2_block_manager:
|
||||
version = "v2"
|
||||
if self.scheduler_config.embedding_mode:
|
||||
version = "embedding"
|
||||
if (self.scheduler_config.embedding_mode
|
||||
or self.cache_config.is_attention_free):
|
||||
version = "placeholder"
|
||||
|
||||
BlockSpaceManagerImpl = BlockSpaceManager.get_block_space_manager_class(
|
||||
version)
|
||||
|
||||
Reference in New Issue
Block a user