[Model] Add LongCat-Flash-Lite (n-gram embedding) (#47857)

Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Michael Goin
2026-07-10 07:17:50 -07:00
committed by GitHub
parent 978a6dfa3f
commit 08dfd68610
16 changed files with 630 additions and 12 deletions
+19 -1
View File
@@ -499,7 +499,7 @@ class LlamaBidirectionalConfig(VerifyAndUpdateConfig):
"last": "LAST",
}
pooling_type = pooling_type_map.get(hf_config.pooling, None)
pooling_type = pooling_type_map.get(hf_config.pooling)
if pooling_type is None:
raise ValueError(f"pool_type {hf_config.pooling!r} not supported")
@@ -809,6 +809,23 @@ class VoyageQwen3BidirectionalEmbedModelConfig(VerifyAndUpdateConfig):
model_config.hf_config.embedding_size = model_config.hf_config.num_labels
class LongcatFlashNgramForCausalLMConfig(VerifyAndUpdateConfig):
@staticmethod
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
# LongCat-Flash-Lite's zero-expert MoE trips a data-dependent assert
# under torch.compile, and its n-gram inputs_embeds are only wired for
# FULL cudagraph capture (PIECEWISE prefill drops them). Default to
# no-compile + FULL cudagraph (prefill runs eager) unless the user
# configured compilation explicitly.
from vllm.config.compilation import CompilationMode, CUDAGraphMode
compilation_config = vllm_config.compilation_config
if compilation_config.mode is None:
compilation_config.mode = CompilationMode.NONE
if compilation_config.cudagraph_mode is None:
compilation_config.cudagraph_mode = CUDAGraphMode.FULL
MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
"ColBERTJinaRobertaModel": JinaRobertaModelConfig,
"ColQwen3_5": ColQwen3_5Config,
@@ -822,6 +839,7 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
"Gemma4ForConditionalGeneration": Gemma4Config,
"Gemma4UnifiedForConditionalGeneration": Gemma4Config,
"GptOssForCausalLM": GptOssForCausalLMConfig,
"LongcatFlashNgramForCausalLM": LongcatFlashNgramForCausalLMConfig,
"GteModel": SnowflakeGteNewModelConfig,
"GteNewForSequenceClassification": GteNewModelConfig,
"GteNewModel": GteNewModelConfig,