[K3] support recoverssm for K3 (#51855)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Benjamin Chislett <bchislett@nvidia.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Benjamin Chislett <bchislett@nvidia.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Benjamin Chislett
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
f27ae25473
commit
70afdedc10
@@ -29,6 +29,7 @@ from vllm.config import (
|
||||
from vllm.config.compilation import CompilationMode, CUDAGraphMode
|
||||
from vllm.config.kernel import IrOpPriorityConfig
|
||||
from vllm.config.load import LoadConfig
|
||||
from vllm.config.mamba import MambaBackendEnum
|
||||
from vllm.config.utils import get_field
|
||||
from vllm.config.vllm import OPTIMIZATION_LEVEL_TO_CONFIG, OptimizationLevel
|
||||
from vllm.platforms import current_platform
|
||||
@@ -37,6 +38,52 @@ from vllm.v1.attention.backend import AttentionCGSupport
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
def test_kda_recoverssm_derivation_is_revalidated():
|
||||
config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(
|
||||
use_replayssm=True,
|
||||
use_kda_recoverssm=False,
|
||||
mamba_cache_mode="none",
|
||||
),
|
||||
num_speculative_tokens=3,
|
||||
model_config=SimpleNamespace(
|
||||
supports_replayssm=True,
|
||||
architecture="KimiLinearForCausalLM",
|
||||
),
|
||||
mamba_config=SimpleNamespace(
|
||||
backend=MambaBackendEnum.TRITON,
|
||||
enable_stochastic_rounding=False,
|
||||
),
|
||||
parallel_config=SimpleNamespace(pipeline_parallel_size=1),
|
||||
kv_transfer_config=None,
|
||||
use_v2_model_runner=True,
|
||||
)
|
||||
|
||||
VllmConfig.validate_mamba_cached_kernel(config)
|
||||
assert config.cache_config.use_replayssm
|
||||
assert config.cache_config.use_kda_recoverssm
|
||||
|
||||
config.cache_config.mamba_cache_mode = "align"
|
||||
VllmConfig.validate_mamba_cached_kernel(config)
|
||||
config.use_v2_model_runner = False
|
||||
with pytest.raises(ValueError, match="VLLM_USE_V2_MODEL_RUNNER=1"):
|
||||
VllmConfig.validate_mamba_cached_kernel(config)
|
||||
config.use_v2_model_runner = True
|
||||
config.cache_config.mamba_cache_mode = "all"
|
||||
with pytest.raises(ValueError, match="only none and align"):
|
||||
VllmConfig.validate_mamba_cached_kernel(config)
|
||||
config.cache_config.mamba_cache_mode = "none"
|
||||
|
||||
config.model_config.architecture = "NemotronHForCausalLM"
|
||||
with pytest.raises(ValueError, match="only supported for Kimi-K3 KDA"):
|
||||
VllmConfig.validate_mamba_cached_kernel(config)
|
||||
|
||||
config.model_config.architecture = "KimiLinearForCausalLM"
|
||||
config.parallel_config.pipeline_parallel_size = 2
|
||||
with pytest.raises(ValueError, match="pipeline_parallel_size=1"):
|
||||
VllmConfig.validate_mamba_cached_kernel(config)
|
||||
|
||||
|
||||
def test_compile_config_repr_succeeds():
|
||||
# setup: VllmBackend mutates the config object
|
||||
config = VllmConfig()
|
||||
|
||||
Reference in New Issue
Block a user