[Perf][DSpark] Add KV-only context insertion across V4.1 cache formats (#56441)

Signed-off-by: 0z5a <0z5a@users.noreply.github.com>
Signed-off-by: Yongye Zhu <yongye@inferact.ai>
Co-authored-by: 0z5a <0z5a@users.noreply.github.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
0z5a
2026-09-15 21:34:55 -07:00
committed by GitHub
co-authored by 0z5a Codex Yongye Zhu Claude Opus 5
parent a31ec3a68b
commit 6ecd97f1b7
5 changed files with 234 additions and 61 deletions
@@ -657,7 +657,12 @@ static void launchFusedDeepseekV4Templated(
config.attrs = attrs;
config.numAttrs = (sm_version >= 90) ? 1 : 0;
if (num_tokens_full < NUM_TOKEN_CUTOFF) {
// The reduced grid pays off by iterating a token's Q heads inside one CTA.
// With zero query heads there is nothing to iterate, so one warp per block
// would do all the work and the other seven idle; keep the warp-per-slot
// grid instead. Only reachable above NUM_TOKEN_CUTOFF tokens in a single
// insert, where it is worth ~2x.
if (kNumHeadsQPadded == 0 || num_tokens_full < NUM_TOKEN_CUTOFF) {
cudaLaunchKernelEx(&config,
fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel<
scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>,
@@ -993,6 +998,86 @@ static void launchFullCacheKernel(
// ────────────────────────────────────────────────────────────────────────────
// Torch op wrapper
// ────────────────────────────────────────────────────────────────────────────
void fused_deepseek_v4_kv_rope_insert(
torch::stable::Tensor const& kv, torch::stable::Tensor& k_cache,
torch::stable::Tensor const& slot_mapping,
torch::stable::Tensor const& position_ids,
torch::stable::Tensor const& cos_sin_cache, int64_t cache_block_size,
std::optional<torch::stable::Tensor> fp8_scale, bool kv_mxfp8) {
using torch::headeronly::ScalarType;
STD_TORCH_CHECK(kv.device().is_cuda() && kv.is_contiguous() &&
kv.dim() == 2 && kv.size(1) == 512 &&
kv.scalar_type() == ScalarType::BFloat16,
"kv must be contiguous CUDA bfloat16 [tokens, 512]");
auto const device = kv.get_device_index();
STD_TORCH_CHECK(k_cache.device().is_cuda() &&
k_cache.get_device_index() == device &&
k_cache.dim() == 3 && cache_block_size > 0 &&
k_cache.size(1) == cache_block_size &&
k_cache.stride(2) == 1,
"k_cache must be a paged cache on the KV device");
for (auto const* tensor : {&slot_mapping, &position_ids}) {
STD_TORCH_CHECK(tensor->device().is_cuda() &&
tensor->get_device_index() == device &&
tensor->scalar_type() == ScalarType::Long &&
tensor->dim() == 1 && tensor->is_contiguous(),
"slots and positions must be contiguous int64 on the KV device");
}
STD_TORCH_CHECK(position_ids.size(0) == kv.size(0) &&
slot_mapping.size(0) <= kv.size(0),
"positions must match KV rows; slots may omit padded rows");
STD_TORCH_CHECK(cos_sin_cache.device().is_cuda() &&
cos_sin_cache.get_device_index() == device &&
cos_sin_cache.scalar_type() == ScalarType::Float &&
cos_sin_cache.dim() == 2 &&
cos_sin_cache.size(1) == 64 && cos_sin_cache.is_contiguous(),
"cos_sin_cache must be contiguous float32 [max_pos, 64]");
bool const packed = k_cache.scalar_type() == ScalarType::Byte;
bool const fp8 = k_cache.scalar_type() == ScalarType::Float8_e4m3fn;
STD_TORCH_CHECK(packed || fp8 ||
k_cache.scalar_type() == ScalarType::BFloat16,
"cache must be uint8, bfloat16 or float8_e4m3fn");
STD_TORCH_CHECK(k_cache.size(2) == (packed ? (kv_mxfp8 ? 528 : 584) : 512),
"cache row width does not match its dtype");
if (fp8) {
STD_TORCH_CHECK(fp8_scale.has_value() && fp8_scale->device().is_cuda() &&
fp8_scale->get_device_index() == device &&
fp8_scale->scalar_type() == ScalarType::Float &&
fp8_scale->numel() == 1,
"fp8 cache requires one float32 KV scale on the KV device");
}
int const num_tokens = static_cast<int>(slot_mapping.size(0));
if (num_tokens == 0) return;
const torch::stable::accelerator::DeviceGuard device_guard(device);
const cudaStream_t stream = get_current_cuda_stream(device);
using scalar_t = c10::BFloat16;
auto const* input = reinterpret_cast<scalar_t const*>(kv.const_data_ptr());
auto* cache = reinterpret_cast<uint8_t*>(k_cache.mutable_data_ptr());
auto const* slots = slot_mapping.const_data_ptr<int64_t>();
auto const* positions = position_ids.const_data_ptr<int64_t>();
auto const* cos_sin = cos_sin_cache.const_data_ptr<float>();
// Zero query heads schedules only the existing KV branch, preserving its
// RoPE rounding and quantization without allocating any query tensors.
if (packed) {
vllm::deepseek_v4_fused_ops::launchFusedDeepseekV4Templated<scalar_t, 0, false>(
nullptr, nullptr, input, cache, slots, positions, cos_sin, 0.0f,
num_tokens, num_tokens, 0, static_cast<int>(cache_block_size),
static_cast<int>(k_cache.stride(0)), kv_mxfp8, stream);
} else if (fp8) {
vllm::deepseek_v4_fused_ops::launchFullCacheKernel<scalar_t, false, true>(
nullptr, nullptr, 0, 0, input, cache, slots, positions, cos_sin,
fp8_scale->const_data_ptr<float>(), nullptr, 0.0f, num_tokens, num_tokens,
0, static_cast<int>(cache_block_size), k_cache.stride(0),
k_cache.stride(1), false, "fused_deepseek_v4_kv_rope_insert", stream);
} else {
vllm::deepseek_v4_fused_ops::launchFullCacheKernel<scalar_t, false, false>(
nullptr, nullptr, 0, 0, input, cache, slots, positions, cos_sin, nullptr,
nullptr, 0.0f, num_tokens, num_tokens, 0, static_cast<int>(cache_block_size),
k_cache.stride(0) * 2, k_cache.stride(1) * 2, false,
"fused_deepseek_v4_kv_rope_insert", stream);
}
}
torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
torch::stable::Tensor const& q_in, // [N, num_heads_q, 512] bf16
torch::stable::Tensor const& kv, // [N, 512] bf16 (read-only)
+7
View File
@@ -259,6 +259,13 @@ void fused_qk_norm_rope(torch::stable::Tensor& qkv, int64_t num_heads_q,
torch::stable::Tensor& position_ids,
int64_t forced_token_heads_per_warp);
void fused_deepseek_v4_kv_rope_insert(
torch::stable::Tensor const& kv, torch::stable::Tensor& k_cache,
torch::stable::Tensor const& slot_mapping,
torch::stable::Tensor const& position_ids,
torch::stable::Tensor const& cos_sin_cache, int64_t cache_block_size,
std::optional<torch::stable::Tensor> fp8_scale, bool kv_mxfp8);
torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
torch::stable::Tensor const& q_in, torch::stable::Tensor const& kv,
torch::stable::Tensor& k_cache, torch::stable::Tensor const& slot_mapping,
+8
View File
@@ -425,6 +425,12 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
"bool is_neox, Tensor position_ids, "
"int forced_token_heads_per_warp=-1) -> ()");
ops.def(
"fused_deepseek_v4_kv_rope_insert("
"Tensor kv, Tensor! k_cache, Tensor slot_mapping, Tensor position_ids, "
"Tensor cos_sin_cache, int cache_block_size, Tensor? fp8_scale=None, "
"bool kv_mxfp8=False) -> "
"()");
ops.def(
"fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert("
"Tensor q_in, Tensor kv, Tensor! k_cache, "
@@ -807,6 +813,8 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
// Positional encoding kernels (shared CUDA/ROCm)
ops.impl("rotary_embedding", TORCH_BOX(&rotary_embedding));
ops.impl("fused_qk_norm_rope", TORCH_BOX(&fused_qk_norm_rope));
ops.impl("fused_deepseek_v4_kv_rope_insert",
TORCH_BOX(&fused_deepseek_v4_kv_rope_insert));
ops.impl("fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert",
TORCH_BOX(&fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert));
ops.impl(