[Model] Add LongCat-Flash-Lite (n-gram embedding) (#47857)

Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Michael Goin
2026-07-10 07:17:50 -07:00
committed by GitHub
parent 978a6dfa3f
commit 08dfd68610
16 changed files with 630 additions and 12 deletions
+1
View File
@@ -390,6 +390,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
"csrc/libtorch_stable/cuda_view.cu"
"csrc/libtorch_stable/cuda_utils_kernels.cu"
"csrc/libtorch_stable/activation_kernels.cu"
"csrc/libtorch_stable/ngram_embedding_kernels.cu"
"csrc/libtorch_stable/quantization/activation_kernels.cu"
"csrc/libtorch_stable/quantization/w8a8/int8/scaled_quant.cu"
"csrc/libtorch_stable/quantization/w8a8/fp8/common.cu"
@@ -0,0 +1,96 @@
// N-gram embedding index kernel for LongCat-Flash (n-gram embedding variant).
//
// Adapted from SGLang:
// https://github.com/sgl-project/sglang/blob/main/python/sglang/jit_kernel/csrc/ngram_embedding.cuh
//
// For each position, computes the hashed n-gram embedding ids that index the
// concatenated embedder table. Integer tensors are int32 except ``row_indices``
// (int64); the token table is ``[max_running_reqs, max_context_len]`` int32,
// where a negative entry marks an ignored token (e.g. an EOS boundary).
#include "torch_utils.h"
#include "ops.h"
#include <cstdint>
namespace vllm::ngram_embedding {
constexpr int kBlockThreads = 256;
__global__ void ComputeNGramIdsKernel(
int batch_size, int ne_n, int ne_k,
int* ne_weights, // [ne_n-1, ne_k, ne_n]
int* ne_mods, // [ne_n-1, ne_k]
int* exclusive_ne_embedder_size_sums, // [(ne_n-1)*ne_k + 1]
int* exclusive_req_len_sums, // [batch_size + 1]
int* ne_token_table, // [max_running_reqs, max_context_len]
int max_context_len,
const int64_t* __restrict__ row_indices, // [batch_size]
int* column_starts, // [batch_size]
int* n_gram_ids // [token_num, (ne_n-1)*ne_k]
) {
const int req_id = blockIdx.x % batch_size;
const int config_id = (blockIdx.x - req_id) / batch_size;
// n and k are offset from their physical meaning: n = real_n - 2, k = real_k
// - 1 (they index into ne_weights / ne_mods).
const int k = config_id % ne_k;
const int n = (config_id - config_id % ne_k) / ne_k;
const int ne_weight_base_idx = n * ne_k * ne_n + k * ne_n;
const int ne_mod = ne_mods[n * ne_k + k];
for (int i = exclusive_req_len_sums[req_id] + threadIdx.x;
i < exclusive_req_len_sums[req_id + 1]; i += blockDim.x) {
uint64_t n_gram_id = 0;
const int64_t current_token_offset = i - exclusive_req_len_sums[req_id];
const int64_t req_token_table_index =
row_indices[req_id] * static_cast<int64_t>(max_context_len);
const int64_t current_token_table_index =
req_token_table_index + column_starts[req_id] + current_token_offset;
for (int j = 0; j < n + 2; j++) {
if (current_token_table_index - j < req_token_table_index) {
break; // outside this request's range
}
if (ne_token_table[current_token_table_index - j] < 0) {
break; // ignored token
}
const uint64_t term =
(uint64_t)ne_token_table[current_token_table_index - j] *
(uint64_t)ne_weights[ne_weight_base_idx + j];
n_gram_id += term % ne_mod;
}
n_gram_id %= ne_mod;
n_gram_id += exclusive_ne_embedder_size_sums[n * ne_k + k];
n_gram_ids[i * (ne_n - 1) * ne_k + n * ne_k + k] = (int)(n_gram_id);
}
}
} // namespace vllm::ngram_embedding
void ngram_compute_n_gram_ids(
int64_t ne_n, int64_t ne_k, torch::stable::Tensor& ne_weights,
torch::stable::Tensor& ne_mods,
torch::stable::Tensor& exclusive_ne_embedder_size_sums,
torch::stable::Tensor& exclusive_req_len_sums,
torch::stable::Tensor& ne_token_table, torch::stable::Tensor& row_indices,
torch::stable::Tensor& column_starts, torch::stable::Tensor& n_gram_ids) {
const int batch_size = static_cast<int>(exclusive_req_len_sums.size(0) - 1);
const int max_context_len = static_cast<int>(ne_token_table.size(1));
const int num_configs = (static_cast<int>(ne_n) - 1) * static_cast<int>(ne_k);
const int grid_size = num_configs * batch_size;
if (grid_size <= 0) return;
const torch::stable::accelerator::DeviceGuard device_guard(
ne_weights.get_device_index());
const cudaStream_t stream = get_current_cuda_stream();
vllm::ngram_embedding::ComputeNGramIdsKernel<<<
grid_size, vllm::ngram_embedding::kBlockThreads, 0, stream>>>(
batch_size, static_cast<int>(ne_n), static_cast<int>(ne_k),
ne_weights.mutable_data_ptr<int32_t>(),
ne_mods.mutable_data_ptr<int32_t>(),
exclusive_ne_embedder_size_sums.mutable_data_ptr<int32_t>(),
exclusive_req_len_sums.mutable_data_ptr<int32_t>(),
ne_token_table.mutable_data_ptr<int32_t>(), max_context_len,
row_indices.const_data_ptr<int64_t>(),
column_starts.mutable_data_ptr<int32_t>(),
n_gram_ids.mutable_data_ptr<int32_t>());
}
+9
View File
@@ -554,3 +554,12 @@ void cp_gather_indexer_k_quant_cache(
// quant_block_size * 4]
const torch::stable::Tensor& block_table, // [batch_size, num_blocks]
const torch::stable::Tensor& cu_seq_lens); // [batch_size + 1]
// LongCat n-gram embedding index kernel (see ngram_embedding_kernels.cu).
void ngram_compute_n_gram_ids(
int64_t ne_n, int64_t ne_k, torch::stable::Tensor& ne_weights,
torch::stable::Tensor& ne_mods,
torch::stable::Tensor& exclusive_ne_embedder_size_sums,
torch::stable::Tensor& exclusive_req_len_sums,
torch::stable::Tensor& ne_token_table, torch::stable::Tensor& row_indices,
torch::stable::Tensor& column_starts, torch::stable::Tensor& n_gram_ids);
+13
View File
@@ -598,9 +598,22 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
"Tensor? initial_state_idx,"
"Tensor? cu_chunk_seqlen,"
"Tensor? last_chunk_indices) -> ()");
// LongCat n-gram embedding index kernel. All tensor args are marked mutable
// to match the (non-const) stable-Tensor& C++ signature; only ne_token_table
// and n_gram_ids are actually written in place.
ops.def(
"ngram_compute_n_gram_ids(int ne_n, int ne_k, Tensor(a!) ne_weights, "
"Tensor(b!) ne_mods, Tensor(c!) exclusive_ne_embedder_size_sums, "
"Tensor(d!) exclusive_req_len_sums, Tensor(e!) ne_token_table, "
"Tensor(f!) row_indices, Tensor(g!) column_starts, "
"Tensor(h!) n_gram_ids) -> ()");
}
STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
// LongCat n-gram embedding index kernel.
ops.impl("ngram_compute_n_gram_ids", TORCH_BOX(&ngram_compute_n_gram_ids));
// Per-token group quantization
ops.impl("per_token_group_fp8_quant", TORCH_BOX(&per_token_group_quant_fp8));
ops.impl("per_token_group_fp8_quant_packed",
+7
View File
@@ -382,6 +382,13 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"LongcatFlashForCausalLM": _HfExamplesInfo(
"meituan-longcat/LongCat-Flash-Chat", trust_remote_code=True
),
"LongcatFlashNgramForCausalLM": _HfExamplesInfo(
"meituan-longcat/LongCat-Flash-Lite",
trust_remote_code=True,
# Shrink the ~62GB n-gram tables (ngram_vocab_size_ratio * vocab_size)
# so the dummy-weight init test fits in CI memory.
hf_overrides={"ngram_vocab_size_ratio": 1},
),
"MambaForCausalLM": _HfExamplesInfo("state-spaces/mamba-130m-hf"),
"Mamba2ForCausalLM": _HfExamplesInfo(
"mistralai/Mamba-Codestral-7B-v0.1",
+7 -2
View File
@@ -529,8 +529,13 @@ def dummy_hf_overrides(
}
)
# Update num_hidden_layers for non-Longcat architectures
if model_arch != "LongcatFlashForCausalLM" and model_arch != "LongCatFlashMTPModel":
# Update num_hidden_layers for non-Longcat architectures (Longcat derives it
# from num_layers for its dual-attention layers).
if model_arch not in (
"LongcatFlashForCausalLM",
"LongCatFlashMTPModel",
"LongcatFlashNgramForCausalLM",
):
update_dict["num_hidden_layers"] = num_hidden_layers
text_config.update(update_dict)
+31
View File
@@ -235,6 +235,37 @@ def rms_norm(
torch.ops._C.rms_norm(out, input, weight, epsilon)
# LongCat n-gram embedding index kernel (see csrc/.../ngram_embedding_kernels.cu).
def ngram_compute_n_gram_ids(
ne_n: int,
ne_k: int,
ne_weights: torch.Tensor,
ne_mods: torch.Tensor,
exclusive_ne_embedder_size_sums: torch.Tensor,
exclusive_req_len_sums: torch.Tensor,
ne_token_table: torch.Tensor,
row_indices: torch.Tensor,
column_starts: torch.Tensor,
n_gram_ids: torch.Tensor,
) -> None:
"""Compute concatenated (offset) n-gram ids for a ragged prefill batch.
Writes ``n_gram_ids`` of shape ``[token_num, (ne_n-1)*ne_k]``.
"""
torch.ops._C.ngram_compute_n_gram_ids(
ne_n,
ne_k,
ne_weights,
ne_mods,
exclusive_ne_embedder_size_sums,
exclusive_req_len_sums,
ne_token_table,
row_indices,
column_starts,
n_gram_ids,
)
def fused_add_rms_norm(
input: torch.Tensor,
residual: torch.Tensor,
+1 -1
View File
@@ -518,7 +518,7 @@ class SpeculativeConfig:
"architectures": ["Qwen3_5MoeMTP" if is_moe else "Qwen3_5MTP"],
}
)
if hf_config.model_type == "longcat_flash":
if hf_config.model_type in ("longcat_flash", "longcat_flash_ngram"):
hf_config.model_type = "longcat_flash_mtp"
n_predict = getattr(hf_config, "num_nextn_predict_layers", 1)
hf_config.update(
+1
View File
@@ -70,6 +70,7 @@ DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES = frozenset(
"DeepseekV2ForCausalLM",
"Qwen2MoeForCausalLM",
"GraniteMoeForCausalLM",
"LongcatFlashNgramForCausalLM",
}
)
+19 -1
View File
@@ -499,7 +499,7 @@ class LlamaBidirectionalConfig(VerifyAndUpdateConfig):
"last": "LAST",
}
pooling_type = pooling_type_map.get(hf_config.pooling, None)
pooling_type = pooling_type_map.get(hf_config.pooling)
if pooling_type is None:
raise ValueError(f"pool_type {hf_config.pooling!r} not supported")
@@ -809,6 +809,23 @@ class VoyageQwen3BidirectionalEmbedModelConfig(VerifyAndUpdateConfig):
model_config.hf_config.embedding_size = model_config.hf_config.num_labels
class LongcatFlashNgramForCausalLMConfig(VerifyAndUpdateConfig):
@staticmethod
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
# LongCat-Flash-Lite's zero-expert MoE trips a data-dependent assert
# under torch.compile, and its n-gram inputs_embeds are only wired for
# FULL cudagraph capture (PIECEWISE prefill drops them). Default to
# no-compile + FULL cudagraph (prefill runs eager) unless the user
# configured compilation explicitly.
from vllm.config.compilation import CompilationMode, CUDAGraphMode
compilation_config = vllm_config.compilation_config
if compilation_config.mode is None:
compilation_config.mode = CompilationMode.NONE
if compilation_config.cudagraph_mode is None:
compilation_config.cudagraph_mode = CUDAGraphMode.FULL
MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
"ColBERTJinaRobertaModel": JinaRobertaModelConfig,
"ColQwen3_5": ColQwen3_5Config,
@@ -822,6 +839,7 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
"Gemma4ForConditionalGeneration": Gemma4Config,
"Gemma4UnifiedForConditionalGeneration": Gemma4Config,
"GptOssForCausalLM": GptOssForCausalLMConfig,
"LongcatFlashNgramForCausalLM": LongcatFlashNgramForCausalLMConfig,
"GteModel": SnowflakeGteNewModelConfig,
"GteNewForSequenceClassification": GteNewModelConfig,
"GteNewModel": GteNewModelConfig,
+12 -3
View File
@@ -318,7 +318,7 @@ class LongcatMoe(nn.Module):
hidden_states = hidden_states.view(-1, hidden_dim)
# Align to FusedMoE padded hidden size to avoid dim mismatch
padded_hidden = self.experts.hidden_size
padded_hidden = self.experts.moe_config.hidden_dim
if hidden_dim < padded_hidden:
hidden_states_padded = torch.nn.functional.pad(
hidden_states,
@@ -687,14 +687,23 @@ class FlashModel(nn.Module):
).split([self_attn.qk_nope_head_dim, self_attn.v_head_dim], dim=1)
self_attn.w_kc = w_kc.transpose(1, 2).contiguous().transpose(1, 2)
self_attn.w_vc = w_vc.contiguous().transpose(1, 2)
if self.config.mla_scale_q_lora:
# Guard against compounding on incremental load_weights calls:
# the in-place ``*=`` would otherwise re-apply the MLA LoRA
# scaling to the layernorm weights on each pass.
if self.config.mla_scale_q_lora and not getattr(
self_attn, "_mla_q_lora_scaled", False
):
self_attn.q_a_layernorm.weight.data *= (
self.config.hidden_size / self.config.q_lora_rank
) ** 0.5
if self.config.mla_scale_kv_lora:
self_attn._mla_q_lora_scaled = True
if self.config.mla_scale_kv_lora and not getattr(
self_attn, "_mla_kv_lora_scaled", False
):
self_attn.kv_a_layernorm.weight.data *= (
self.config.hidden_size / self.config.kv_lora_rank
) ** 0.5
self_attn._mla_kv_lora_scaled = True
return loaded_params
@@ -126,8 +126,10 @@ class LongCatMultiTokenPredictor(nn.Module):
class LongCatFlashMTP(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
# LongCat MTP without MoE layers
vllm_config.model_config.hf_config.n_routed_experts = None
# LongCat MTP has no MoE layers: clear n_routed_experts so the predictor
# builds a dense MLP. object.__setattr__ bypasses the ngram remote
# config's strict validation (it rejects setting the int field to None).
object.__setattr__(vllm_config.model_config.hf_config, "n_routed_experts", None)
self.config = FlashConfig(**vllm_config.model_config.hf_config.__dict__)
self.quant_config = (
None
@@ -287,14 +289,22 @@ class LongCatFlashMTP(nn.Module):
).split([self_attn.qk_nope_head_dim, self_attn.v_head_dim], dim=1)
self_attn.w_kc = w_kc.transpose(1, 2).contiguous().transpose(1, 2)
self_attn.w_vc = w_vc.contiguous().transpose(1, 2)
if self.config.mla_scale_q_lora:
# Guard against compounding on incremental load_weights calls (the
# in-place *= would otherwise double-apply the LoRA scaling).
if self.config.mla_scale_q_lora and not getattr(
self_attn, "_mla_q_lora_scaled", False
):
self_attn.q_a_layernorm.weight.data *= (
self.config.hidden_size / self.config.q_lora_rank
) ** 0.5
if self.config.mla_scale_kv_lora:
self_attn._mla_q_lora_scaled = True
if self.config.mla_scale_kv_lora and not getattr(
self_attn, "_mla_kv_lora_scaled", False
):
self_attn.kv_a_layernorm.weight.data *= (
self.config.hidden_size / self.config.kv_lora_rank
) ** 0.5
self_attn._mla_kv_lora_scaled = True
return loaded_params
def _rewrite_spec_layer_name(
@@ -0,0 +1,405 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inference-only LongCat-Flash-Lite (n-gram embedding) model.
``LongcatFlashNgramForCausalLM`` is LongCat-Flash (MLA dual-attention +
zero-expert MoE + YaRN) plus an n-gram embedding input layer: each position's
embedding fuses the token embedding with hashed embeddings of the preceding
``n`` tokens. That per-request token history is isolated in a Model-Runner-V2
:class:`LongcatNgramModelState` (mirroring ``DiffusionGemmaModelState``), so
``get_model_state_cls`` makes the model MRV2-only.
"""
from collections.abc import Iterable
from typing import Any
import torch
from torch import nn
from vllm import _custom_ops as ops
from vllm.config import VllmConfig
from vllm.distributed import get_pp_group
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.v1.core.sched.output import NewRequestData
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.model_states.default import DefaultModelState
from vllm.v1.worker.gpu.states import RequestState
from .interfaces import SupportsLoRA, SupportsPP
from .longcat_flash import FlashConfig, FlashModel
from .utils import AutoWeightsLoader, PPMissingLayer, maybe_prefix
def uses_ngram_embedding(config: FlashConfig) -> bool:
return getattr(config, "ngram_vocab_size_ratio", None) is not None
def _config_dtype(config: FlashConfig) -> torch.dtype:
dt = getattr(config, "torch_dtype", None) or getattr(config, "dtype", None)
if isinstance(dt, torch.dtype):
return dt
return getattr(torch, str(dt), None) or torch.bfloat16
class NgramEmbedding(nn.Module):
"""Token embedding fused with hashed n-gram embeddings.
TP-sharded: the ``k*(n-1)`` per-embedder tables are concatenated into one
:class:`VocabParallelEmbedding` (``oe_embedder``) with per-embedder offsets,
and the projections are stacked into one ``oe_projection`` applied with a
single ``bmm``. Hashing math is ported from the HF reference.
"""
def __init__(self, config: FlashConfig, base_embeddings: nn.Module) -> None:
super().__init__()
self.config = config
self.word_embeddings = base_embeddings
self.m = config.ngram_vocab_size_ratio * config.vocab_size
self.k = config.emb_split_num
self.n = config.emb_neighbor_num
self.pad_id = config.pad_token_id
self.eos_token_id = config.eos_token_id
self._dtype = _config_dtype(config)
self._init_ngram_embeddings()
def _init_ngram_embeddings(self) -> None:
self.num_embedders = self.k * (self.n - 1)
oe_dim = self.config.hidden_size // self.num_embedders
self.oe_dim = oe_dim
# Exclusive prefix sums of per-embedder table sizes; each embedder's
# local id is offset into the single concatenated table.
sizes = [int(self.m + i * 2 + 1) for i in range(self.num_embedders)]
offsets = [0]
for s in sizes:
offsets.append(offsets[-1] + s)
self._offsets = offsets # len num_embedders + 1
self._sizes = sizes
self.oe_embedder = VocabParallelEmbedding(
offsets[-1], oe_dim, params_dtype=self._dtype
)
# Stacked projections: oe_projection[i] = post_projs[i].weight.T
self.oe_projection = nn.Parameter(
torch.empty(
self.num_embedders, oe_dim, self.config.hidden_size, dtype=self._dtype
),
requires_grad=False,
)
# Precomputed tables for the CUDA n-gram id kernel (ngram_embedding
# _kernels.cu): ne_weights[i][j][delta] = vocab^delta mod ne_mods[i][j],
# ne_mods[i][j] = m + 2*(i*k+j) + 1. Registered as non-persistent buffers
# so they follow the module to the device (not part of the checkpoint).
vocab = self.config.vocab_size
ne_weights = torch.zeros(self.n - 1, self.k, self.n, dtype=torch.int32)
ne_mods = torch.zeros(self.n - 1, self.k, dtype=torch.int32)
for i in range(self.n - 1):
for j in range(self.k):
mod = int(self.m + 2 * (i * self.k + j) + 1)
ne_mods[i, j] = mod
for delta in range(self.n):
ne_weights[i, j, delta] = pow(vocab, delta, mod)
self.register_buffer("ne_weights", ne_weights, persistent=False)
self.register_buffer("ne_mods", ne_mods, persistent=False)
self.register_buffer(
"exclusive_sizes",
torch.tensor(offsets, dtype=torch.int32),
persistent=False,
)
def load_weight(self, weight_name: str, loaded_weight: torch.Tensor) -> str:
"""Split a per-embedder checkpoint weight into the sharded layout.
Returns the destination parameter's qualified name (relative to the
enclosing model) so the caller can mark it loaded for completeness
checks.
"""
if "ngram_embeddings.embedders." in weight_name:
index = int(
weight_name.split("ngram_embeddings.embedders.")[1].split(".")[0]
)
lo, hi = self._offsets[index], self._offsets[index + 1]
assert hi - lo == loaded_weight.shape[0], (
f"{hi - lo=} {loaded_weight.shape[0]=}"
)
shard = self.oe_embedder.shard_indices
tp_start, tp_end = shard.org_vocab_start_index, shard.org_vocab_end_index
load_start, load_end = max(lo, tp_start), min(hi, tp_end)
if load_start < load_end:
self.oe_embedder.weight.data[
load_start - tp_start : load_end - tp_start
] = loaded_weight[load_start - lo : load_end - lo]
return "ngram_embeddings.oe_embedder.weight"
elif "ngram_embeddings.post_projs." in weight_name:
index = int(
weight_name.split("ngram_embeddings.post_projs.")[1].split(".")[0]
)
self.oe_projection.data[index].copy_(loaded_weight.t())
return "ngram_embeddings.oe_projection"
else:
raise AssertionError(f"Unexpected ngram weight: {weight_name}")
def embed_batched(
self, input_ids: torch.Tensor, oe_ids: torch.Tensor
) -> torch.Tensor:
"""Fused n-gram embedding for a flat batch given precomputed ids.
Args:
input_ids: ``[num_tokens]`` current token per position.
oe_ids: ``[num_tokens, num_embedders]`` global (offset) n-gram ids,
as produced by the ``ngram_compute_n_gram_ids`` kernel.
Returns: ``[num_tokens, hidden]``.
"""
word = self.word_embeddings(input_ids) # [N, H]
flat = oe_ids.permute(1, 0).contiguous() # [num_embedders, N]
oe = self.oe_embedder(flat) # [num_embedders, N, oe_dim]
proj = torch.bmm(oe, self.oe_projection) # [num_embedders, N, H]
all_h = torch.cat([word.unsqueeze(0), proj], dim=0) # [ne+1, N, H]
return all_h.mean(dim=0) # [N, H]
class FlashNgramModel(FlashModel):
"""FlashModel whose input embedding is an :class:`NgramEmbedding`."""
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
# Each FlashDecoderLayer is a *dual* layer (2 attentions), so the number
# of decoder layers is ``num_layers``. The ngram HF config sets
# ``num_hidden_layers`` to a multiple of that (attention-module count),
# which FlashModel would otherwise build as too many (dead) layers.
hf = vllm_config.model_config.hf_config
num_layers = getattr(hf, "num_layers", None)
if num_layers is not None and hf.num_hidden_layers != num_layers:
hf.num_hidden_layers = num_layers
super().__init__(vllm_config=vllm_config, prefix=prefix)
if get_pp_group().is_first_rank and uses_ngram_embedding(self.config):
self.ngram_embeddings = NgramEmbedding(self.config, self.embed_tokens)
else:
self.ngram_embeddings = None
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
# Names arrive with the ``model.`` prefix already stripped (routed here
# by AutoWeightsLoader). Split the concatenated/sharded ngram tables and
# stacked projections; delegate everything else to FlashModel.
loaded: set[str] = set()
rest: list[tuple[str, torch.Tensor]] = []
for name, w in weights:
if self.ngram_embeddings is not None and (
"ngram_embeddings.embedders." in name
or "ngram_embeddings.post_projs." in name
):
loaded.add(self.ngram_embeddings.load_weight(name, w))
else:
rest.append((name, w))
loaded |= super().load_weights(rest)
return loaded
class LongcatFlashNgramForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
"""LongCat-Flash-Lite for causal LM (MRV2-only, n-gram embedding)."""
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
"gate_up_proj": ["gate_proj", "up_proj"],
}
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
if not vllm_config.use_v2_model_runner:
raise NotImplementedError(
"LongcatFlashNgramForCausalLM (LongCat-Flash-Lite) requires the "
"V2 model runner for its n-gram embedding state; it is selected "
"automatically unless VLLM_USE_V2_MODEL_RUNNER=0 is set."
)
config = FlashConfig(**vllm_config.model_config.hf_config.__dict__)
config.intermediate_size = getattr(
config, "ffn_hidden_size", config.intermediate_size
)
self.config = config
self.quant_config = vllm_config.quant_config
self.model = FlashNgramModel(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
if get_pp_group().is_last_rank:
self.lm_head = ParallelLMHead(
config.vocab_size,
config.hidden_size,
quant_config=self.quant_config,
prefix=maybe_prefix(prefix, "lm_head"),
)
else:
self.lm_head = PPMissingLayer()
self.logits_processor = LogitsProcessor(config.vocab_size)
self.make_empty_intermediate_tensors = (
self.model.make_empty_intermediate_tensors
)
@staticmethod
def get_model_state_cls() -> type["LongcatNgramModelState"]:
return LongcatNgramModelState
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors=None,
inputs_embeds: torch.Tensor | None = None,
):
# inputs_embeds is produced by LongcatNgramModelState.prepare_inputs.
return self.model(input_ids, positions, intermediate_tensors, inputs_embeds)
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
return self.logits_processor(self.lm_head, hidden_states)
def get_expert_mapping(self):
return self.model.get_expert_mapping()
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
# AutoWeightsLoader routes ``model.*`` to FlashNgramModel.load_weights
# (which handles the ngram split) and ``lm_head.*`` to the head. MTP
# weights are not part of this model.
loader = AutoWeightsLoader(self, skip_prefixes=["model.mtp."])
return loader.load_weights(weights)
class LongcatNgramModelState(DefaultModelState):
"""Per-request n-gram token history for LongCat-Flash-Lite.
Maintains a small CPU-side per-slot context (last ``n-1`` processed tokens)
and a persistent ``inputs_embeds`` buffer. ``prepare_inputs`` computes the
fused n-gram embedding per request into the buffer, handed to the model
forward as ``inputs_embeds``.
"""
def __init__(self, vllm_config, model, encoder_cache, device) -> None:
super().__init__(vllm_config, model, encoder_cache, device)
config = model.config
self.ngram = model.model.ngram_embeddings
self.n = int(config.emb_neighbor_num)
self.ctx_len = self.n - 1
self.eos_id = int(config.eos_token_id)
# Per-slot left-context: last ``n-1`` processed tokens, EOS negated. A
# negative entry (incl. the -1 fill) marks a context boundary that stops
# the n-gram walk (matches the kernel's EOS break / fresh-request start).
self.token_context = torch.full(
(self.max_num_reqs, self.ctx_len), -1, dtype=torch.int32, device=device
)
self._inputs_embeds_buf = torch.zeros(
self.max_num_tokens,
config.hidden_size,
dtype=self.dtype,
device=device,
)
def _neg_eos(self, toks: list[int]) -> list[int]:
return [-t if t == self.eos_id else t for t in toks]
def add_request(self, req_index: int, new_req_data: NewRequestData) -> None:
super().add_request(req_index, new_req_data) # rope positions
# Fresh request -> no left-context (-1 fill). On resume, seed from the
# already-processed token tail. Use prefill_token_ids (full processed
# sequence incl. generated tokens on v2 resume), like DefaultModelState;
# the prompt alone would be too short when resuming after decode.
ctx = [-1] * self.ctx_len
ncomp = new_req_data.num_computed_tokens
toks_src = new_req_data.prefill_token_ids or new_req_data.prompt_token_ids
if ncomp > 0 and toks_src is not None:
lo = max(0, ncomp - self.ctx_len)
toks = self._neg_eos(list(toks_src[lo:ncomp]))
ctx[self.ctx_len - len(toks) :] = toks
self.token_context[req_index] = torch.tensor(
ctx, dtype=torch.int32, device=self.token_context.device
)
def prepare_inputs(
self, input_batch: InputBatch, req_states: RequestState
) -> dict[str, Any]:
model_inputs = super().prepare_inputs(input_batch, req_states) # positions
num_tokens = input_batch.num_tokens
num_padded = input_batch.num_tokens_after_padding
input_ids = input_batch.input_ids[:num_tokens]
embeds = self._inputs_embeds_buf[:num_padded]
oe_ids = self._compute_oe_ids(input_batch)
embeds[:num_tokens].copy_(self.ngram.embed_batched(input_ids, oe_ids))
model_inputs["inputs_embeds"] = embeds
return model_inputs
def prepare_dummy_inputs(self, num_reqs: int, num_tokens: int) -> dict[str, Any]:
# FULL cudagraph replay reads only the captured buffers, so capture must
# reference the same persistent ``inputs_embeds`` buffer prepare_inputs
# re-fills (the base class wires this for multimodal models only).
model_inputs = super().prepare_dummy_inputs(num_reqs, num_tokens) # positions
model_inputs["inputs_embeds"] = self._inputs_embeds_buf[:num_tokens]
return model_inputs
def _compute_oe_ids(self, input_batch: InputBatch) -> torch.Tensor:
"""Batched global n-gram ids ``[num_tokens, num_embedders]``.
Assembles an ephemeral per-request token table (``[n-1] context ++
current tokens``, EOS-negated) and runs the ``ngram_compute_n_gram_ids``
CUDA kernel for the whole batch, then rolls each slot's context forward.
"""
device = self.token_context.device
num_tokens = input_batch.num_tokens
num_reqs = input_batch.num_reqs
ctx_len = self.ctx_len
idx_mapping = input_batch.idx_mapping[:num_reqs].long()
qsl = input_batch.query_start_loc[: num_reqs + 1].to(torch.int32)
cur = input_batch.input_ids[:num_tokens].to(torch.int32)
cur_neg = torch.where(cur == self.eos_id, -cur, cur)
req_lens = qsl[1:] - qsl[:-1]
max_len = int(req_lens.max().item())
width = ctx_len + max_len
# table[r] = [context(n-1) | current tokens | pad(-1)]
table = torch.full((num_reqs, width), -1, dtype=torch.int32, device=device)
table[:, :ctx_len] = self.token_context[idx_mapping]
tok_req = torch.repeat_interleave(
torch.arange(num_reqs, device=device), req_lens.long()
)
col = ctx_len + (
torch.arange(num_tokens, device=device) - qsl[:-1].long()[tok_req]
)
table[tok_req, col] = cur_neg
column_starts = torch.full(
(num_reqs,), ctx_len, dtype=torch.int32, device=device
)
row_indices = torch.arange(num_reqs, dtype=torch.int64, device=device)
n_gram_ids = torch.empty(
num_tokens, self.ngram.num_embedders, dtype=torch.int32, device=device
)
ops.ngram_compute_n_gram_ids(
self.n,
self.ngram.k,
self.ngram.ne_weights,
self.ngram.ne_mods,
self.ngram.exclusive_sizes,
qsl,
table,
row_indices,
column_starts,
n_gram_ids,
)
# Roll context: new context = last n-1 of [context | current] per slot.
gather = req_lens.long().unsqueeze(1) + torch.arange(
ctx_len, device=device
).unsqueeze(0)
rows = torch.arange(num_reqs, device=device).unsqueeze(1)
self.token_context[idx_mapping] = table[rows, gather]
return n_gram_ids.long()
+4
View File
@@ -145,6 +145,10 @@ _TEXT_GENERATION_MODELS = {
# For decapoda-research/llama-*
"LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
"LongcatFlashForCausalLM": ("longcat_flash", "LongcatFlashForCausalLM"),
"LongcatFlashNgramForCausalLM": (
"longcat_flash_ngram",
"LongcatFlashNgramForCausalLM",
),
"MambaForCausalLM": ("mamba", "MambaForCausalLM"),
"Mamba2ForCausalLM": ("mamba2", "Mamba2ForCausalLM"),
"MellumForCausalLM": ("mellum", "MellumForCausalLM"),
@@ -268,6 +268,7 @@ class ModelArchConfigConvertorBase:
"kimi_k2",
"kimi_linear",
"longcat_flash",
"longcat_flash_ngram",
"pangu_ultra_moe",
"pangu_ultra_moe_mtp",
"bailing_hybrid",
+9 -1
View File
@@ -540,7 +540,15 @@ def init_kv_cache(
shared_kv_cache_layers=shared_kv_cache_layers,
kv_cache_config=kv_cache_config,
)
bind_kv_cache(kv_caches, forward_context, runner_kv_caches)
# Dual-attention models (e.g. LongCat-Flash) put two Attention modules per
# decoder layer, so a layer name carries two integers (layer + module index).
num_attn_module = (
2
if vllm_config.model_config.hf_config.model_type
in ("longcat_flash", "longcat_flash_ngram")
else 1
)
bind_kv_cache(kv_caches, forward_context, runner_kv_caches, num_attn_module)
return kv_caches