[Model] Add LongCat-Flash-Lite (n-gram embedding) (#47857)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -499,7 +499,7 @@ class LlamaBidirectionalConfig(VerifyAndUpdateConfig):
|
||||
"last": "LAST",
|
||||
}
|
||||
|
||||
pooling_type = pooling_type_map.get(hf_config.pooling, None)
|
||||
pooling_type = pooling_type_map.get(hf_config.pooling)
|
||||
if pooling_type is None:
|
||||
raise ValueError(f"pool_type {hf_config.pooling!r} not supported")
|
||||
|
||||
@@ -809,6 +809,23 @@ class VoyageQwen3BidirectionalEmbedModelConfig(VerifyAndUpdateConfig):
|
||||
model_config.hf_config.embedding_size = model_config.hf_config.num_labels
|
||||
|
||||
|
||||
class LongcatFlashNgramForCausalLMConfig(VerifyAndUpdateConfig):
|
||||
@staticmethod
|
||||
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
|
||||
# LongCat-Flash-Lite's zero-expert MoE trips a data-dependent assert
|
||||
# under torch.compile, and its n-gram inputs_embeds are only wired for
|
||||
# FULL cudagraph capture (PIECEWISE prefill drops them). Default to
|
||||
# no-compile + FULL cudagraph (prefill runs eager) unless the user
|
||||
# configured compilation explicitly.
|
||||
from vllm.config.compilation import CompilationMode, CUDAGraphMode
|
||||
|
||||
compilation_config = vllm_config.compilation_config
|
||||
if compilation_config.mode is None:
|
||||
compilation_config.mode = CompilationMode.NONE
|
||||
if compilation_config.cudagraph_mode is None:
|
||||
compilation_config.cudagraph_mode = CUDAGraphMode.FULL
|
||||
|
||||
|
||||
MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
|
||||
"ColBERTJinaRobertaModel": JinaRobertaModelConfig,
|
||||
"ColQwen3_5": ColQwen3_5Config,
|
||||
@@ -822,6 +839,7 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
|
||||
"Gemma4ForConditionalGeneration": Gemma4Config,
|
||||
"Gemma4UnifiedForConditionalGeneration": Gemma4Config,
|
||||
"GptOssForCausalLM": GptOssForCausalLMConfig,
|
||||
"LongcatFlashNgramForCausalLM": LongcatFlashNgramForCausalLMConfig,
|
||||
"GteModel": SnowflakeGteNewModelConfig,
|
||||
"GteNewForSequenceClassification": GteNewModelConfig,
|
||||
"GteNewModel": GteNewModelConfig,
|
||||
|
||||
Reference in New Issue
Block a user