DCP supports hybrid attention (#40996)
Signed-off-by: YanXu <yancey.yx@alibaba-inc.com> Signed-off-by: Jingyi Yang <girasoleyang@gmail.com> Co-authored-by: Jingyi Yang <girasoleyang@gmail.com>
This commit is contained in:
@@ -43,6 +43,11 @@ HYBRID_MODELS = [
|
||||
"tiny-random/qwen3-next-moe",
|
||||
]
|
||||
|
||||
HYBRID_MODELS_REQUIRING_CHUNKED_PREFILL = {
|
||||
"LiquidAI/LFM2-1.2B",
|
||||
"tiny-random/qwen3-next-moe",
|
||||
}
|
||||
|
||||
FULL_CUDA_GRAPH_MODELS = [
|
||||
"ai21labs/Jamba-tiny-dev",
|
||||
"pfnet/plamo-2-1b",
|
||||
@@ -92,8 +97,15 @@ def test_models(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
extra_kwargs = {}
|
||||
if model in HYBRID_MODELS_REQUIRING_CHUNKED_PREFILL:
|
||||
extra_kwargs["enable_chunked_prefill"] = True
|
||||
|
||||
with vllm_runner(
|
||||
model, max_num_seqs=MAX_NUM_SEQS, attention_backend=ATTN_BACKEND
|
||||
model,
|
||||
max_num_seqs=MAX_NUM_SEQS,
|
||||
attention_backend=ATTN_BACKEND,
|
||||
**extra_kwargs,
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
|
||||
@@ -171,6 +171,7 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
|
||||
"Describe this video in one sentence."
|
||||
),
|
||||
needs_video_metadata=True,
|
||||
vllm_runner_kwargs={"enable_chunked_prefill": True},
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
"internvl": VitCudagraphTestConfig(
|
||||
|
||||
Reference in New Issue
Block a user