diff --git a/CMakeLists.txt b/CMakeLists.txt index 7fc49a2d15..3ddce1c3c3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -390,6 +390,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") "csrc/libtorch_stable/cuda_view.cu" "csrc/libtorch_stable/cuda_utils_kernels.cu" "csrc/libtorch_stable/activation_kernels.cu" + "csrc/libtorch_stable/ngram_embedding_kernels.cu" "csrc/libtorch_stable/quantization/activation_kernels.cu" "csrc/libtorch_stable/quantization/w8a8/int8/scaled_quant.cu" "csrc/libtorch_stable/quantization/w8a8/fp8/common.cu" diff --git a/csrc/libtorch_stable/ngram_embedding_kernels.cu b/csrc/libtorch_stable/ngram_embedding_kernels.cu new file mode 100644 index 0000000000..a4da62f9ac --- /dev/null +++ b/csrc/libtorch_stable/ngram_embedding_kernels.cu @@ -0,0 +1,96 @@ +// N-gram embedding index kernel for LongCat-Flash (n-gram embedding variant). +// +// Adapted from SGLang: +// https://github.com/sgl-project/sglang/blob/main/python/sglang/jit_kernel/csrc/ngram_embedding.cuh +// +// For each position, computes the hashed n-gram embedding ids that index the +// concatenated embedder table. Integer tensors are int32 except ``row_indices`` +// (int64); the token table is ``[max_running_reqs, max_context_len]`` int32, +// where a negative entry marks an ignored token (e.g. an EOS boundary). + +#include "torch_utils.h" + +#include "ops.h" + +#include + +namespace vllm::ngram_embedding { + +constexpr int kBlockThreads = 256; + +__global__ void ComputeNGramIdsKernel( + int batch_size, int ne_n, int ne_k, + int* ne_weights, // [ne_n-1, ne_k, ne_n] + int* ne_mods, // [ne_n-1, ne_k] + int* exclusive_ne_embedder_size_sums, // [(ne_n-1)*ne_k + 1] + int* exclusive_req_len_sums, // [batch_size + 1] + int* ne_token_table, // [max_running_reqs, max_context_len] + int max_context_len, + const int64_t* __restrict__ row_indices, // [batch_size] + int* column_starts, // [batch_size] + int* n_gram_ids // [token_num, (ne_n-1)*ne_k] +) { + const int req_id = blockIdx.x % batch_size; + const int config_id = (blockIdx.x - req_id) / batch_size; + // n and k are offset from their physical meaning: n = real_n - 2, k = real_k + // - 1 (they index into ne_weights / ne_mods). + const int k = config_id % ne_k; + const int n = (config_id - config_id % ne_k) / ne_k; + const int ne_weight_base_idx = n * ne_k * ne_n + k * ne_n; + const int ne_mod = ne_mods[n * ne_k + k]; + for (int i = exclusive_req_len_sums[req_id] + threadIdx.x; + i < exclusive_req_len_sums[req_id + 1]; i += blockDim.x) { + uint64_t n_gram_id = 0; + const int64_t current_token_offset = i - exclusive_req_len_sums[req_id]; + const int64_t req_token_table_index = + row_indices[req_id] * static_cast(max_context_len); + const int64_t current_token_table_index = + req_token_table_index + column_starts[req_id] + current_token_offset; + for (int j = 0; j < n + 2; j++) { + if (current_token_table_index - j < req_token_table_index) { + break; // outside this request's range + } + if (ne_token_table[current_token_table_index - j] < 0) { + break; // ignored token + } + const uint64_t term = + (uint64_t)ne_token_table[current_token_table_index - j] * + (uint64_t)ne_weights[ne_weight_base_idx + j]; + n_gram_id += term % ne_mod; + } + n_gram_id %= ne_mod; + n_gram_id += exclusive_ne_embedder_size_sums[n * ne_k + k]; + n_gram_ids[i * (ne_n - 1) * ne_k + n * ne_k + k] = (int)(n_gram_id); + } +} + +} // namespace vllm::ngram_embedding + +void ngram_compute_n_gram_ids( + int64_t ne_n, int64_t ne_k, torch::stable::Tensor& ne_weights, + torch::stable::Tensor& ne_mods, + torch::stable::Tensor& exclusive_ne_embedder_size_sums, + torch::stable::Tensor& exclusive_req_len_sums, + torch::stable::Tensor& ne_token_table, torch::stable::Tensor& row_indices, + torch::stable::Tensor& column_starts, torch::stable::Tensor& n_gram_ids) { + const int batch_size = static_cast(exclusive_req_len_sums.size(0) - 1); + const int max_context_len = static_cast(ne_token_table.size(1)); + const int num_configs = (static_cast(ne_n) - 1) * static_cast(ne_k); + const int grid_size = num_configs * batch_size; + if (grid_size <= 0) return; + + const torch::stable::accelerator::DeviceGuard device_guard( + ne_weights.get_device_index()); + const cudaStream_t stream = get_current_cuda_stream(); + vllm::ngram_embedding::ComputeNGramIdsKernel<<< + grid_size, vllm::ngram_embedding::kBlockThreads, 0, stream>>>( + batch_size, static_cast(ne_n), static_cast(ne_k), + ne_weights.mutable_data_ptr(), + ne_mods.mutable_data_ptr(), + exclusive_ne_embedder_size_sums.mutable_data_ptr(), + exclusive_req_len_sums.mutable_data_ptr(), + ne_token_table.mutable_data_ptr(), max_context_len, + row_indices.const_data_ptr(), + column_starts.mutable_data_ptr(), + n_gram_ids.mutable_data_ptr()); +} diff --git a/csrc/libtorch_stable/ops.h b/csrc/libtorch_stable/ops.h index 0daa024a76..4520f4a99e 100644 --- a/csrc/libtorch_stable/ops.h +++ b/csrc/libtorch_stable/ops.h @@ -554,3 +554,12 @@ void cp_gather_indexer_k_quant_cache( // quant_block_size * 4] const torch::stable::Tensor& block_table, // [batch_size, num_blocks] const torch::stable::Tensor& cu_seq_lens); // [batch_size + 1] + +// LongCat n-gram embedding index kernel (see ngram_embedding_kernels.cu). +void ngram_compute_n_gram_ids( + int64_t ne_n, int64_t ne_k, torch::stable::Tensor& ne_weights, + torch::stable::Tensor& ne_mods, + torch::stable::Tensor& exclusive_ne_embedder_size_sums, + torch::stable::Tensor& exclusive_req_len_sums, + torch::stable::Tensor& ne_token_table, torch::stable::Tensor& row_indices, + torch::stable::Tensor& column_starts, torch::stable::Tensor& n_gram_ids); diff --git a/csrc/libtorch_stable/torch_bindings.cpp b/csrc/libtorch_stable/torch_bindings.cpp index 66a93a5ba1..7581769f4f 100644 --- a/csrc/libtorch_stable/torch_bindings.cpp +++ b/csrc/libtorch_stable/torch_bindings.cpp @@ -598,9 +598,22 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) { "Tensor? initial_state_idx," "Tensor? cu_chunk_seqlen," "Tensor? last_chunk_indices) -> ()"); + + // LongCat n-gram embedding index kernel. All tensor args are marked mutable + // to match the (non-const) stable-Tensor& C++ signature; only ne_token_table + // and n_gram_ids are actually written in place. + ops.def( + "ngram_compute_n_gram_ids(int ne_n, int ne_k, Tensor(a!) ne_weights, " + "Tensor(b!) ne_mods, Tensor(c!) exclusive_ne_embedder_size_sums, " + "Tensor(d!) exclusive_req_len_sums, Tensor(e!) ne_token_table, " + "Tensor(f!) row_indices, Tensor(g!) column_starts, " + "Tensor(h!) n_gram_ids) -> ()"); } STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) { + // LongCat n-gram embedding index kernel. + ops.impl("ngram_compute_n_gram_ids", TORCH_BOX(&ngram_compute_n_gram_ids)); + // Per-token group quantization ops.impl("per_token_group_fp8_quant", TORCH_BOX(&per_token_group_quant_fp8)); ops.impl("per_token_group_fp8_quant_packed", diff --git a/tests/models/registry.py b/tests/models/registry.py index fccfb549f5..ec9ef35436 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -382,6 +382,13 @@ _TEXT_GENERATION_EXAMPLE_MODELS = { "LongcatFlashForCausalLM": _HfExamplesInfo( "meituan-longcat/LongCat-Flash-Chat", trust_remote_code=True ), + "LongcatFlashNgramForCausalLM": _HfExamplesInfo( + "meituan-longcat/LongCat-Flash-Lite", + trust_remote_code=True, + # Shrink the ~62GB n-gram tables (ngram_vocab_size_ratio * vocab_size) + # so the dummy-weight init test fits in CI memory. + hf_overrides={"ngram_vocab_size_ratio": 1}, + ), "MambaForCausalLM": _HfExamplesInfo("state-spaces/mamba-130m-hf"), "Mamba2ForCausalLM": _HfExamplesInfo( "mistralai/Mamba-Codestral-7B-v0.1", diff --git a/tests/models/utils.py b/tests/models/utils.py index 91d76d5f24..f1018efe8a 100644 --- a/tests/models/utils.py +++ b/tests/models/utils.py @@ -529,8 +529,13 @@ def dummy_hf_overrides( } ) - # Update num_hidden_layers for non-Longcat architectures - if model_arch != "LongcatFlashForCausalLM" and model_arch != "LongCatFlashMTPModel": + # Update num_hidden_layers for non-Longcat architectures (Longcat derives it + # from num_layers for its dual-attention layers). + if model_arch not in ( + "LongcatFlashForCausalLM", + "LongCatFlashMTPModel", + "LongcatFlashNgramForCausalLM", + ): update_dict["num_hidden_layers"] = num_hidden_layers text_config.update(update_dict) diff --git a/vllm/_custom_ops.py b/vllm/_custom_ops.py index 4de43e96a8..2a4579ae03 100644 --- a/vllm/_custom_ops.py +++ b/vllm/_custom_ops.py @@ -235,6 +235,37 @@ def rms_norm( torch.ops._C.rms_norm(out, input, weight, epsilon) +# LongCat n-gram embedding index kernel (see csrc/.../ngram_embedding_kernels.cu). +def ngram_compute_n_gram_ids( + ne_n: int, + ne_k: int, + ne_weights: torch.Tensor, + ne_mods: torch.Tensor, + exclusive_ne_embedder_size_sums: torch.Tensor, + exclusive_req_len_sums: torch.Tensor, + ne_token_table: torch.Tensor, + row_indices: torch.Tensor, + column_starts: torch.Tensor, + n_gram_ids: torch.Tensor, +) -> None: + """Compute concatenated (offset) n-gram ids for a ragged prefill batch. + + Writes ``n_gram_ids`` of shape ``[token_num, (ne_n-1)*ne_k]``. + """ + torch.ops._C.ngram_compute_n_gram_ids( + ne_n, + ne_k, + ne_weights, + ne_mods, + exclusive_ne_embedder_size_sums, + exclusive_req_len_sums, + ne_token_table, + row_indices, + column_starts, + n_gram_ids, + ) + + def fused_add_rms_norm( input: torch.Tensor, residual: torch.Tensor, diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index 3dc8d33cf9..ea749091da 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -518,7 +518,7 @@ class SpeculativeConfig: "architectures": ["Qwen3_5MoeMTP" if is_moe else "Qwen3_5MTP"], } ) - if hf_config.model_type == "longcat_flash": + if hf_config.model_type in ("longcat_flash", "longcat_flash_ngram"): hf_config.model_type = "longcat_flash_mtp" n_predict = getattr(hf_config, "num_nextn_predict_layers", 1) hf_config.update( diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 062877b169..dc01800341 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -70,6 +70,7 @@ DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES = frozenset( "DeepseekV2ForCausalLM", "Qwen2MoeForCausalLM", "GraniteMoeForCausalLM", + "LongcatFlashNgramForCausalLM", } ) diff --git a/vllm/model_executor/models/config.py b/vllm/model_executor/models/config.py index cb01a12efd..96232237a0 100644 --- a/vllm/model_executor/models/config.py +++ b/vllm/model_executor/models/config.py @@ -499,7 +499,7 @@ class LlamaBidirectionalConfig(VerifyAndUpdateConfig): "last": "LAST", } - pooling_type = pooling_type_map.get(hf_config.pooling, None) + pooling_type = pooling_type_map.get(hf_config.pooling) if pooling_type is None: raise ValueError(f"pool_type {hf_config.pooling!r} not supported") @@ -809,6 +809,23 @@ class VoyageQwen3BidirectionalEmbedModelConfig(VerifyAndUpdateConfig): model_config.hf_config.embedding_size = model_config.hf_config.num_labels +class LongcatFlashNgramForCausalLMConfig(VerifyAndUpdateConfig): + @staticmethod + def verify_and_update_config(vllm_config: "VllmConfig") -> None: + # LongCat-Flash-Lite's zero-expert MoE trips a data-dependent assert + # under torch.compile, and its n-gram inputs_embeds are only wired for + # FULL cudagraph capture (PIECEWISE prefill drops them). Default to + # no-compile + FULL cudagraph (prefill runs eager) unless the user + # configured compilation explicitly. + from vllm.config.compilation import CompilationMode, CUDAGraphMode + + compilation_config = vllm_config.compilation_config + if compilation_config.mode is None: + compilation_config.mode = CompilationMode.NONE + if compilation_config.cudagraph_mode is None: + compilation_config.cudagraph_mode = CUDAGraphMode.FULL + + MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = { "ColBERTJinaRobertaModel": JinaRobertaModelConfig, "ColQwen3_5": ColQwen3_5Config, @@ -822,6 +839,7 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = { "Gemma4ForConditionalGeneration": Gemma4Config, "Gemma4UnifiedForConditionalGeneration": Gemma4Config, "GptOssForCausalLM": GptOssForCausalLMConfig, + "LongcatFlashNgramForCausalLM": LongcatFlashNgramForCausalLMConfig, "GteModel": SnowflakeGteNewModelConfig, "GteNewForSequenceClassification": GteNewModelConfig, "GteNewModel": GteNewModelConfig, diff --git a/vllm/model_executor/models/longcat_flash.py b/vllm/model_executor/models/longcat_flash.py index 3dd1118aa8..18628a64ce 100644 --- a/vllm/model_executor/models/longcat_flash.py +++ b/vllm/model_executor/models/longcat_flash.py @@ -318,7 +318,7 @@ class LongcatMoe(nn.Module): hidden_states = hidden_states.view(-1, hidden_dim) # Align to FusedMoE padded hidden size to avoid dim mismatch - padded_hidden = self.experts.hidden_size + padded_hidden = self.experts.moe_config.hidden_dim if hidden_dim < padded_hidden: hidden_states_padded = torch.nn.functional.pad( hidden_states, @@ -687,14 +687,23 @@ class FlashModel(nn.Module): ).split([self_attn.qk_nope_head_dim, self_attn.v_head_dim], dim=1) self_attn.w_kc = w_kc.transpose(1, 2).contiguous().transpose(1, 2) self_attn.w_vc = w_vc.contiguous().transpose(1, 2) - if self.config.mla_scale_q_lora: + # Guard against compounding on incremental load_weights calls: + # the in-place ``*=`` would otherwise re-apply the MLA LoRA + # scaling to the layernorm weights on each pass. + if self.config.mla_scale_q_lora and not getattr( + self_attn, "_mla_q_lora_scaled", False + ): self_attn.q_a_layernorm.weight.data *= ( self.config.hidden_size / self.config.q_lora_rank ) ** 0.5 - if self.config.mla_scale_kv_lora: + self_attn._mla_q_lora_scaled = True + if self.config.mla_scale_kv_lora and not getattr( + self_attn, "_mla_kv_lora_scaled", False + ): self_attn.kv_a_layernorm.weight.data *= ( self.config.hidden_size / self.config.kv_lora_rank ) ** 0.5 + self_attn._mla_kv_lora_scaled = True return loaded_params diff --git a/vllm/model_executor/models/longcat_flash_mtp.py b/vllm/model_executor/models/longcat_flash_mtp.py index 13921d7351..26fff05da6 100644 --- a/vllm/model_executor/models/longcat_flash_mtp.py +++ b/vllm/model_executor/models/longcat_flash_mtp.py @@ -126,8 +126,10 @@ class LongCatMultiTokenPredictor(nn.Module): class LongCatFlashMTP(nn.Module): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() - # LongCat MTP without MoE layers - vllm_config.model_config.hf_config.n_routed_experts = None + # LongCat MTP has no MoE layers: clear n_routed_experts so the predictor + # builds a dense MLP. object.__setattr__ bypasses the ngram remote + # config's strict validation (it rejects setting the int field to None). + object.__setattr__(vllm_config.model_config.hf_config, "n_routed_experts", None) self.config = FlashConfig(**vllm_config.model_config.hf_config.__dict__) self.quant_config = ( None @@ -287,14 +289,22 @@ class LongCatFlashMTP(nn.Module): ).split([self_attn.qk_nope_head_dim, self_attn.v_head_dim], dim=1) self_attn.w_kc = w_kc.transpose(1, 2).contiguous().transpose(1, 2) self_attn.w_vc = w_vc.contiguous().transpose(1, 2) - if self.config.mla_scale_q_lora: + # Guard against compounding on incremental load_weights calls (the + # in-place *= would otherwise double-apply the LoRA scaling). + if self.config.mla_scale_q_lora and not getattr( + self_attn, "_mla_q_lora_scaled", False + ): self_attn.q_a_layernorm.weight.data *= ( self.config.hidden_size / self.config.q_lora_rank ) ** 0.5 - if self.config.mla_scale_kv_lora: + self_attn._mla_q_lora_scaled = True + if self.config.mla_scale_kv_lora and not getattr( + self_attn, "_mla_kv_lora_scaled", False + ): self_attn.kv_a_layernorm.weight.data *= ( self.config.hidden_size / self.config.kv_lora_rank ) ** 0.5 + self_attn._mla_kv_lora_scaled = True return loaded_params def _rewrite_spec_layer_name( diff --git a/vllm/model_executor/models/longcat_flash_ngram.py b/vllm/model_executor/models/longcat_flash_ngram.py new file mode 100644 index 0000000000..5aaa1aad9b --- /dev/null +++ b/vllm/model_executor/models/longcat_flash_ngram.py @@ -0,0 +1,405 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inference-only LongCat-Flash-Lite (n-gram embedding) model. + +``LongcatFlashNgramForCausalLM`` is LongCat-Flash (MLA dual-attention + +zero-expert MoE + YaRN) plus an n-gram embedding input layer: each position's +embedding fuses the token embedding with hashed embeddings of the preceding +``n`` tokens. That per-request token history is isolated in a Model-Runner-V2 +:class:`LongcatNgramModelState` (mirroring ``DiffusionGemmaModelState``), so +``get_model_state_cls`` makes the model MRV2-only. +""" + +from collections.abc import Iterable +from typing import Any + +import torch +from torch import nn + +from vllm import _custom_ops as ops +from vllm.config import VllmConfig +from vllm.distributed import get_pp_group +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, + VocabParallelEmbedding, +) +from vllm.v1.core.sched.output import NewRequestData +from vllm.v1.worker.gpu.input_batch import InputBatch +from vllm.v1.worker.gpu.model_states.default import DefaultModelState +from vllm.v1.worker.gpu.states import RequestState + +from .interfaces import SupportsLoRA, SupportsPP +from .longcat_flash import FlashConfig, FlashModel +from .utils import AutoWeightsLoader, PPMissingLayer, maybe_prefix + + +def uses_ngram_embedding(config: FlashConfig) -> bool: + return getattr(config, "ngram_vocab_size_ratio", None) is not None + + +def _config_dtype(config: FlashConfig) -> torch.dtype: + dt = getattr(config, "torch_dtype", None) or getattr(config, "dtype", None) + if isinstance(dt, torch.dtype): + return dt + return getattr(torch, str(dt), None) or torch.bfloat16 + + +class NgramEmbedding(nn.Module): + """Token embedding fused with hashed n-gram embeddings. + + TP-sharded: the ``k*(n-1)`` per-embedder tables are concatenated into one + :class:`VocabParallelEmbedding` (``oe_embedder``) with per-embedder offsets, + and the projections are stacked into one ``oe_projection`` applied with a + single ``bmm``. Hashing math is ported from the HF reference. + """ + + def __init__(self, config: FlashConfig, base_embeddings: nn.Module) -> None: + super().__init__() + self.config = config + self.word_embeddings = base_embeddings + + self.m = config.ngram_vocab_size_ratio * config.vocab_size + self.k = config.emb_split_num + self.n = config.emb_neighbor_num + self.pad_id = config.pad_token_id + self.eos_token_id = config.eos_token_id + self._dtype = _config_dtype(config) + + self._init_ngram_embeddings() + + def _init_ngram_embeddings(self) -> None: + self.num_embedders = self.k * (self.n - 1) + oe_dim = self.config.hidden_size // self.num_embedders + self.oe_dim = oe_dim + + # Exclusive prefix sums of per-embedder table sizes; each embedder's + # local id is offset into the single concatenated table. + sizes = [int(self.m + i * 2 + 1) for i in range(self.num_embedders)] + offsets = [0] + for s in sizes: + offsets.append(offsets[-1] + s) + self._offsets = offsets # len num_embedders + 1 + self._sizes = sizes + + self.oe_embedder = VocabParallelEmbedding( + offsets[-1], oe_dim, params_dtype=self._dtype + ) + # Stacked projections: oe_projection[i] = post_projs[i].weight.T + self.oe_projection = nn.Parameter( + torch.empty( + self.num_embedders, oe_dim, self.config.hidden_size, dtype=self._dtype + ), + requires_grad=False, + ) + + # Precomputed tables for the CUDA n-gram id kernel (ngram_embedding + # _kernels.cu): ne_weights[i][j][delta] = vocab^delta mod ne_mods[i][j], + # ne_mods[i][j] = m + 2*(i*k+j) + 1. Registered as non-persistent buffers + # so they follow the module to the device (not part of the checkpoint). + vocab = self.config.vocab_size + ne_weights = torch.zeros(self.n - 1, self.k, self.n, dtype=torch.int32) + ne_mods = torch.zeros(self.n - 1, self.k, dtype=torch.int32) + for i in range(self.n - 1): + for j in range(self.k): + mod = int(self.m + 2 * (i * self.k + j) + 1) + ne_mods[i, j] = mod + for delta in range(self.n): + ne_weights[i, j, delta] = pow(vocab, delta, mod) + self.register_buffer("ne_weights", ne_weights, persistent=False) + self.register_buffer("ne_mods", ne_mods, persistent=False) + self.register_buffer( + "exclusive_sizes", + torch.tensor(offsets, dtype=torch.int32), + persistent=False, + ) + + def load_weight(self, weight_name: str, loaded_weight: torch.Tensor) -> str: + """Split a per-embedder checkpoint weight into the sharded layout. + + Returns the destination parameter's qualified name (relative to the + enclosing model) so the caller can mark it loaded for completeness + checks. + """ + if "ngram_embeddings.embedders." in weight_name: + index = int( + weight_name.split("ngram_embeddings.embedders.")[1].split(".")[0] + ) + lo, hi = self._offsets[index], self._offsets[index + 1] + assert hi - lo == loaded_weight.shape[0], ( + f"{hi - lo=} {loaded_weight.shape[0]=}" + ) + shard = self.oe_embedder.shard_indices + tp_start, tp_end = shard.org_vocab_start_index, shard.org_vocab_end_index + load_start, load_end = max(lo, tp_start), min(hi, tp_end) + if load_start < load_end: + self.oe_embedder.weight.data[ + load_start - tp_start : load_end - tp_start + ] = loaded_weight[load_start - lo : load_end - lo] + return "ngram_embeddings.oe_embedder.weight" + elif "ngram_embeddings.post_projs." in weight_name: + index = int( + weight_name.split("ngram_embeddings.post_projs.")[1].split(".")[0] + ) + self.oe_projection.data[index].copy_(loaded_weight.t()) + return "ngram_embeddings.oe_projection" + else: + raise AssertionError(f"Unexpected ngram weight: {weight_name}") + + def embed_batched( + self, input_ids: torch.Tensor, oe_ids: torch.Tensor + ) -> torch.Tensor: + """Fused n-gram embedding for a flat batch given precomputed ids. + + Args: + input_ids: ``[num_tokens]`` current token per position. + oe_ids: ``[num_tokens, num_embedders]`` global (offset) n-gram ids, + as produced by the ``ngram_compute_n_gram_ids`` kernel. + Returns: ``[num_tokens, hidden]``. + """ + word = self.word_embeddings(input_ids) # [N, H] + flat = oe_ids.permute(1, 0).contiguous() # [num_embedders, N] + oe = self.oe_embedder(flat) # [num_embedders, N, oe_dim] + proj = torch.bmm(oe, self.oe_projection) # [num_embedders, N, H] + all_h = torch.cat([word.unsqueeze(0), proj], dim=0) # [ne+1, N, H] + return all_h.mean(dim=0) # [N, H] + + +class FlashNgramModel(FlashModel): + """FlashModel whose input embedding is an :class:`NgramEmbedding`.""" + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + # Each FlashDecoderLayer is a *dual* layer (2 attentions), so the number + # of decoder layers is ``num_layers``. The ngram HF config sets + # ``num_hidden_layers`` to a multiple of that (attention-module count), + # which FlashModel would otherwise build as too many (dead) layers. + hf = vllm_config.model_config.hf_config + num_layers = getattr(hf, "num_layers", None) + if num_layers is not None and hf.num_hidden_layers != num_layers: + hf.num_hidden_layers = num_layers + super().__init__(vllm_config=vllm_config, prefix=prefix) + if get_pp_group().is_first_rank and uses_ngram_embedding(self.config): + self.ngram_embeddings = NgramEmbedding(self.config, self.embed_tokens) + else: + self.ngram_embeddings = None + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + # Names arrive with the ``model.`` prefix already stripped (routed here + # by AutoWeightsLoader). Split the concatenated/sharded ngram tables and + # stacked projections; delegate everything else to FlashModel. + loaded: set[str] = set() + rest: list[tuple[str, torch.Tensor]] = [] + for name, w in weights: + if self.ngram_embeddings is not None and ( + "ngram_embeddings.embedders." in name + or "ngram_embeddings.post_projs." in name + ): + loaded.add(self.ngram_embeddings.load_weight(name, w)) + else: + rest.append((name, w)) + loaded |= super().load_weights(rest) + return loaded + + +class LongcatFlashNgramForCausalLM(nn.Module, SupportsLoRA, SupportsPP): + """LongCat-Flash-Lite for causal LM (MRV2-only, n-gram embedding).""" + + packed_modules_mapping = { + "qkv_proj": ["q_proj", "k_proj", "v_proj"], + "gate_up_proj": ["gate_proj", "up_proj"], + } + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + super().__init__() + if not vllm_config.use_v2_model_runner: + raise NotImplementedError( + "LongcatFlashNgramForCausalLM (LongCat-Flash-Lite) requires the " + "V2 model runner for its n-gram embedding state; it is selected " + "automatically unless VLLM_USE_V2_MODEL_RUNNER=0 is set." + ) + config = FlashConfig(**vllm_config.model_config.hf_config.__dict__) + config.intermediate_size = getattr( + config, "ffn_hidden_size", config.intermediate_size + ) + self.config = config + self.quant_config = vllm_config.quant_config + + self.model = FlashNgramModel( + vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") + ) + if get_pp_group().is_last_rank: + self.lm_head = ParallelLMHead( + config.vocab_size, + config.hidden_size, + quant_config=self.quant_config, + prefix=maybe_prefix(prefix, "lm_head"), + ) + else: + self.lm_head = PPMissingLayer() + self.logits_processor = LogitsProcessor(config.vocab_size) + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors + ) + + @staticmethod + def get_model_state_cls() -> type["LongcatNgramModelState"]: + return LongcatNgramModelState + + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.model.embed_input_ids(input_ids) + + def forward( + self, + input_ids: torch.Tensor | None, + positions: torch.Tensor, + intermediate_tensors=None, + inputs_embeds: torch.Tensor | None = None, + ): + # inputs_embeds is produced by LongcatNgramModelState.prepare_inputs. + return self.model(input_ids, positions, intermediate_tensors, inputs_embeds) + + def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None: + return self.logits_processor(self.lm_head, hidden_states) + + def get_expert_mapping(self): + return self.model.get_expert_mapping() + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + # AutoWeightsLoader routes ``model.*`` to FlashNgramModel.load_weights + # (which handles the ngram split) and ``lm_head.*`` to the head. MTP + # weights are not part of this model. + loader = AutoWeightsLoader(self, skip_prefixes=["model.mtp."]) + return loader.load_weights(weights) + + +class LongcatNgramModelState(DefaultModelState): + """Per-request n-gram token history for LongCat-Flash-Lite. + + Maintains a small CPU-side per-slot context (last ``n-1`` processed tokens) + and a persistent ``inputs_embeds`` buffer. ``prepare_inputs`` computes the + fused n-gram embedding per request into the buffer, handed to the model + forward as ``inputs_embeds``. + """ + + def __init__(self, vllm_config, model, encoder_cache, device) -> None: + super().__init__(vllm_config, model, encoder_cache, device) + config = model.config + self.ngram = model.model.ngram_embeddings + self.n = int(config.emb_neighbor_num) + self.ctx_len = self.n - 1 + self.eos_id = int(config.eos_token_id) + + # Per-slot left-context: last ``n-1`` processed tokens, EOS negated. A + # negative entry (incl. the -1 fill) marks a context boundary that stops + # the n-gram walk (matches the kernel's EOS break / fresh-request start). + self.token_context = torch.full( + (self.max_num_reqs, self.ctx_len), -1, dtype=torch.int32, device=device + ) + + self._inputs_embeds_buf = torch.zeros( + self.max_num_tokens, + config.hidden_size, + dtype=self.dtype, + device=device, + ) + + def _neg_eos(self, toks: list[int]) -> list[int]: + return [-t if t == self.eos_id else t for t in toks] + + def add_request(self, req_index: int, new_req_data: NewRequestData) -> None: + super().add_request(req_index, new_req_data) # rope positions + # Fresh request -> no left-context (-1 fill). On resume, seed from the + # already-processed token tail. Use prefill_token_ids (full processed + # sequence incl. generated tokens on v2 resume), like DefaultModelState; + # the prompt alone would be too short when resuming after decode. + ctx = [-1] * self.ctx_len + ncomp = new_req_data.num_computed_tokens + toks_src = new_req_data.prefill_token_ids or new_req_data.prompt_token_ids + if ncomp > 0 and toks_src is not None: + lo = max(0, ncomp - self.ctx_len) + toks = self._neg_eos(list(toks_src[lo:ncomp])) + ctx[self.ctx_len - len(toks) :] = toks + self.token_context[req_index] = torch.tensor( + ctx, dtype=torch.int32, device=self.token_context.device + ) + + def prepare_inputs( + self, input_batch: InputBatch, req_states: RequestState + ) -> dict[str, Any]: + model_inputs = super().prepare_inputs(input_batch, req_states) # positions + num_tokens = input_batch.num_tokens + num_padded = input_batch.num_tokens_after_padding + input_ids = input_batch.input_ids[:num_tokens] + embeds = self._inputs_embeds_buf[:num_padded] + + oe_ids = self._compute_oe_ids(input_batch) + embeds[:num_tokens].copy_(self.ngram.embed_batched(input_ids, oe_ids)) + model_inputs["inputs_embeds"] = embeds + return model_inputs + + def prepare_dummy_inputs(self, num_reqs: int, num_tokens: int) -> dict[str, Any]: + # FULL cudagraph replay reads only the captured buffers, so capture must + # reference the same persistent ``inputs_embeds`` buffer prepare_inputs + # re-fills (the base class wires this for multimodal models only). + model_inputs = super().prepare_dummy_inputs(num_reqs, num_tokens) # positions + model_inputs["inputs_embeds"] = self._inputs_embeds_buf[:num_tokens] + return model_inputs + + def _compute_oe_ids(self, input_batch: InputBatch) -> torch.Tensor: + """Batched global n-gram ids ``[num_tokens, num_embedders]``. + + Assembles an ephemeral per-request token table (``[n-1] context ++ + current tokens``, EOS-negated) and runs the ``ngram_compute_n_gram_ids`` + CUDA kernel for the whole batch, then rolls each slot's context forward. + """ + device = self.token_context.device + num_tokens = input_batch.num_tokens + num_reqs = input_batch.num_reqs + ctx_len = self.ctx_len + idx_mapping = input_batch.idx_mapping[:num_reqs].long() + qsl = input_batch.query_start_loc[: num_reqs + 1].to(torch.int32) + cur = input_batch.input_ids[:num_tokens].to(torch.int32) + + cur_neg = torch.where(cur == self.eos_id, -cur, cur) + req_lens = qsl[1:] - qsl[:-1] + max_len = int(req_lens.max().item()) + width = ctx_len + max_len + + # table[r] = [context(n-1) | current tokens | pad(-1)] + table = torch.full((num_reqs, width), -1, dtype=torch.int32, device=device) + table[:, :ctx_len] = self.token_context[idx_mapping] + tok_req = torch.repeat_interleave( + torch.arange(num_reqs, device=device), req_lens.long() + ) + col = ctx_len + ( + torch.arange(num_tokens, device=device) - qsl[:-1].long()[tok_req] + ) + table[tok_req, col] = cur_neg + + column_starts = torch.full( + (num_reqs,), ctx_len, dtype=torch.int32, device=device + ) + row_indices = torch.arange(num_reqs, dtype=torch.int64, device=device) + n_gram_ids = torch.empty( + num_tokens, self.ngram.num_embedders, dtype=torch.int32, device=device + ) + ops.ngram_compute_n_gram_ids( + self.n, + self.ngram.k, + self.ngram.ne_weights, + self.ngram.ne_mods, + self.ngram.exclusive_sizes, + qsl, + table, + row_indices, + column_starts, + n_gram_ids, + ) + + # Roll context: new context = last n-1 of [context | current] per slot. + gather = req_lens.long().unsqueeze(1) + torch.arange( + ctx_len, device=device + ).unsqueeze(0) + rows = torch.arange(num_reqs, device=device).unsqueeze(1) + self.token_context[idx_mapping] = table[rows, gather] + return n_gram_ids.long() diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 6d70afc4f6..4999662dd6 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -145,6 +145,10 @@ _TEXT_GENERATION_MODELS = { # For decapoda-research/llama-* "LLaMAForCausalLM": ("llama", "LlamaForCausalLM"), "LongcatFlashForCausalLM": ("longcat_flash", "LongcatFlashForCausalLM"), + "LongcatFlashNgramForCausalLM": ( + "longcat_flash_ngram", + "LongcatFlashNgramForCausalLM", + ), "MambaForCausalLM": ("mamba", "MambaForCausalLM"), "Mamba2ForCausalLM": ("mamba2", "Mamba2ForCausalLM"), "MellumForCausalLM": ("mellum", "MellumForCausalLM"), diff --git a/vllm/transformers_utils/model_arch_config_convertor.py b/vllm/transformers_utils/model_arch_config_convertor.py index 7ced083afb..b6f73f39f3 100644 --- a/vllm/transformers_utils/model_arch_config_convertor.py +++ b/vllm/transformers_utils/model_arch_config_convertor.py @@ -268,6 +268,7 @@ class ModelArchConfigConvertorBase: "kimi_k2", "kimi_linear", "longcat_flash", + "longcat_flash_ngram", "pangu_ultra_moe", "pangu_ultra_moe_mtp", "bailing_hybrid", diff --git a/vllm/v1/worker/gpu/attn_utils.py b/vllm/v1/worker/gpu/attn_utils.py index becfd610bd..227c51bb47 100644 --- a/vllm/v1/worker/gpu/attn_utils.py +++ b/vllm/v1/worker/gpu/attn_utils.py @@ -540,7 +540,15 @@ def init_kv_cache( shared_kv_cache_layers=shared_kv_cache_layers, kv_cache_config=kv_cache_config, ) - bind_kv_cache(kv_caches, forward_context, runner_kv_caches) + # Dual-attention models (e.g. LongCat-Flash) put two Attention modules per + # decoder layer, so a layer name carries two integers (layer + module index). + num_attn_module = ( + 2 + if vllm_config.model_config.hf_config.model_type + in ("longcat_flash", "longcat_flash_ngram") + else 1 + ) + bind_kv_cache(kv_caches, forward_context, runner_kv_caches, num_attn_module) return kv_caches