[Bugfix] Fix FlashInfer non-causal draft attention (DFlash/DSpark) on Blackwell (#48167)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
49e777cf08
commit
ecf4aa5ce2
@@ -0,0 +1,55 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Config-only resolution of DFlash draft attention causality.
|
||||||
|
|
||||||
|
``dflash_has_any_non_causal`` decides pre-build whether the draft needs a
|
||||||
|
non-causal-capable backend, so its branch table (explicit override, SWA-derived
|
||||||
|
per-layer causality, and the no-``layer_types`` fallback) is worth pinning.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from vllm.model_executor.models.qwen3_dflash import (
|
||||||
|
_dflash_layer_causal,
|
||||||
|
dflash_has_any_non_causal,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _config(num_hidden_layers, layer_types=None, causal_override=None):
|
||||||
|
dflash_config = None if causal_override is None else {"causal": causal_override}
|
||||||
|
return SimpleNamespace(
|
||||||
|
num_hidden_layers=num_hidden_layers,
|
||||||
|
layer_types=layer_types,
|
||||||
|
dflash_config=dflash_config,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"config,expected",
|
||||||
|
[
|
||||||
|
# Override forces causality on every layer, ignoring layer_types.
|
||||||
|
(_config(2, layer_types=["full_attention"] * 2, causal_override=True), False),
|
||||||
|
# Override forces non-causal on every layer.
|
||||||
|
(
|
||||||
|
_config(2, layer_types=["sliding_attention"] * 2, causal_override=False),
|
||||||
|
True,
|
||||||
|
),
|
||||||
|
# SWA-derived: full-attention layers are non-causal.
|
||||||
|
(_config(2, layer_types=["sliding_attention", "full_attention"]), True),
|
||||||
|
# SWA-derived: all-sliding is fully causal.
|
||||||
|
(_config(2, layer_types=["sliding_attention", "sliding_attention"]), False),
|
||||||
|
# No layer_types -> non-causal fallback.
|
||||||
|
(_config(2, layer_types=None), True),
|
||||||
|
(_config(2, layer_types=[]), True),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_dflash_has_any_non_causal(config, expected):
|
||||||
|
assert dflash_has_any_non_causal(config) is expected
|
||||||
|
|
||||||
|
|
||||||
|
def test_dflash_layer_causal_is_per_layer():
|
||||||
|
config = _config(2, layer_types=["sliding_attention", "full_attention"])
|
||||||
|
assert _dflash_layer_causal(config, 0) is True
|
||||||
|
assert _dflash_layer_causal(config, 1) is False
|
||||||
@@ -1480,7 +1480,9 @@ def _get_backends_from_return(stmts: list) -> list[str]:
|
|||||||
|
|
||||||
|
|
||||||
def _is_sm100_check(test: ast.expr) -> bool:
|
def _is_sm100_check(test: ast.expr) -> bool:
|
||||||
"""Check if test is `something.major == 10`."""
|
"""Check if test is `something.major == 10`, possibly inside an `and`."""
|
||||||
|
if isinstance(test, ast.BoolOp) and isinstance(test.op, ast.And):
|
||||||
|
return any(_is_sm100_check(value) for value in test.values)
|
||||||
return (
|
return (
|
||||||
isinstance(test, ast.Compare)
|
isinstance(test, ast.Compare)
|
||||||
and isinstance(test.left, ast.Attribute)
|
and isinstance(test.left, ast.Attribute)
|
||||||
|
|||||||
@@ -52,6 +52,26 @@ from .utils import (
|
|||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
_SLIDING_ATTENTION = "sliding_attention"
|
||||||
|
|
||||||
|
|
||||||
|
def _dflash_layer_causal(config: Qwen3Config, layer_idx: int) -> bool:
|
||||||
|
"""``dflash_config.causal`` overrides all layers; else only SWA layers causal."""
|
||||||
|
override = (getattr(config, "dflash_config", None) or {}).get("causal")
|
||||||
|
if override is not None:
|
||||||
|
return override
|
||||||
|
layer_types = getattr(config, "layer_types", None)
|
||||||
|
return bool(layer_types) and layer_types[layer_idx] == _SLIDING_ATTENTION
|
||||||
|
|
||||||
|
|
||||||
|
def dflash_has_any_non_causal(config: Qwen3Config) -> bool:
|
||||||
|
"""Whether the draft needs a non-causal-capable backend, resolved from config
|
||||||
|
(config mirror of the model's ``get_draft_attn_causal``, usable pre-build)."""
|
||||||
|
return not all(
|
||||||
|
_dflash_layer_causal(config, i) for i in range(config.num_hidden_layers)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _resolve_layer_attention(
|
def _resolve_layer_attention(
|
||||||
config: Qwen3Config, layer_idx: int
|
config: Qwen3Config, layer_idx: int
|
||||||
) -> tuple[int | None, bool]:
|
) -> tuple[int | None, bool]:
|
||||||
@@ -79,12 +99,10 @@ def _resolve_layer_attention(
|
|||||||
dflash_config = getattr(config, "dflash_config", None) or {}
|
dflash_config = getattr(config, "dflash_config", None) or {}
|
||||||
layer_types = getattr(config, "layer_types", None)
|
layer_types = getattr(config, "layer_types", None)
|
||||||
use_swa = dflash_config.get("use_swa", False)
|
use_swa = dflash_config.get("use_swa", False)
|
||||||
config_causal = dflash_config.get("causal", None)
|
|
||||||
|
|
||||||
SLIDING_ATTENTION = "sliding_attention"
|
|
||||||
any_sliding = False
|
any_sliding = False
|
||||||
if layer_types is not None:
|
if layer_types is not None:
|
||||||
num_sliding = sum(lt == SLIDING_ATTENTION for lt in layer_types)
|
num_sliding = sum(lt == _SLIDING_ATTENTION for lt in layer_types)
|
||||||
any_sliding = num_sliding > 0
|
any_sliding = num_sliding > 0
|
||||||
# Mixed sliding/full attention needs multiple KV groups (V2 runner only).
|
# Mixed sliding/full attention needs multiple KV groups (V2 runner only).
|
||||||
if (
|
if (
|
||||||
@@ -97,16 +115,11 @@ def _resolve_layer_attention(
|
|||||||
"VLLM_USE_V2_MODEL_RUNNER=1."
|
"VLLM_USE_V2_MODEL_RUNNER=1."
|
||||||
)
|
)
|
||||||
|
|
||||||
default_causal = False
|
# ``use_swa`` forces SWA on every layer, even an all-full ``layer_types``.
|
||||||
if layer_types is None or (use_swa and not any_sliding):
|
if layer_types is None or (use_swa and not any_sliding):
|
||||||
# An absent ``layer_types`` (or the all-"full_attention" one that may
|
|
||||||
# be synthesized when the checkpoint omits it) must not override
|
|
||||||
# ``dflash_config.use_swa``, which forces SWA on every layer.
|
|
||||||
is_sliding = use_swa
|
is_sliding = use_swa
|
||||||
else:
|
else:
|
||||||
is_sliding = layer_types[layer_idx] == SLIDING_ATTENTION
|
is_sliding = layer_types[layer_idx] == _SLIDING_ATTENTION
|
||||||
# Full-attention layers default non-causal; SWA layers default causal.
|
|
||||||
default_causal = is_sliding
|
|
||||||
|
|
||||||
sliding_window = None
|
sliding_window = None
|
||||||
if is_sliding:
|
if is_sliding:
|
||||||
@@ -119,8 +132,7 @@ def _resolve_layer_attention(
|
|||||||
"dflash_config.swa_window_size or the top-level sliding_window."
|
"dflash_config.swa_window_size or the top-level sliding_window."
|
||||||
)
|
)
|
||||||
|
|
||||||
causal = config_causal if config_causal is not None else default_causal
|
return sliding_window, _dflash_layer_causal(config, layer_idx)
|
||||||
return sliding_window, causal
|
|
||||||
|
|
||||||
|
|
||||||
class DFlashQwen3Attention(nn.Module):
|
class DFlashQwen3Attention(nn.Module):
|
||||||
|
|||||||
@@ -85,6 +85,7 @@ def _get_backend_priorities(
|
|||||||
device_capability: DeviceCapability,
|
device_capability: DeviceCapability,
|
||||||
num_heads: int | None = None,
|
num_heads: int | None = None,
|
||||||
kv_cache_dtype: CacheDType | None = None,
|
kv_cache_dtype: CacheDType | None = None,
|
||||||
|
use_non_causal: bool = False,
|
||||||
) -> list[AttentionBackendEnum]:
|
) -> list[AttentionBackendEnum]:
|
||||||
"""Get backend priorities with lazy import to avoid circular dependency."""
|
"""Get backend priorities with lazy import to avoid circular dependency."""
|
||||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||||
@@ -141,7 +142,10 @@ def _get_backend_priorities(
|
|||||||
AttentionBackendEnum.FLASHMLA_SPARSE,
|
AttentionBackendEnum.FLASHMLA_SPARSE,
|
||||||
]
|
]
|
||||||
else:
|
else:
|
||||||
if device_capability.major == 10:
|
# SM100f defaults to FlashInfer for TRTLLM causal attention, but its non-causal
|
||||||
|
# cutlass path (used for dflash attention) is known to have problems.
|
||||||
|
# So prefer FlashAttention when non-causal on SM100f.
|
||||||
|
if device_capability.major == 10 and not use_non_causal:
|
||||||
return [
|
return [
|
||||||
AttentionBackendEnum.FLASHINFER,
|
AttentionBackendEnum.FLASHINFER,
|
||||||
AttentionBackendEnum.FLASH_ATTN,
|
AttentionBackendEnum.FLASH_ATTN,
|
||||||
@@ -368,6 +372,7 @@ class CudaPlatformBase(Platform):
|
|||||||
device_capability,
|
device_capability,
|
||||||
num_heads,
|
num_heads,
|
||||||
attn_selector_config.kv_cache_dtype,
|
attn_selector_config.kv_cache_dtype,
|
||||||
|
attn_selector_config.use_non_causal,
|
||||||
)
|
)
|
||||||
for priority, backend in enumerate(backend_priorities):
|
for priority, backend in enumerate(backend_priorities):
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -927,7 +927,8 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
|||||||
has_trtllm_support = False
|
has_trtllm_support = False
|
||||||
break
|
break
|
||||||
|
|
||||||
if has_trtllm_support:
|
# trtllm-gen only supports causal attention.
|
||||||
|
if has_trtllm_support and not vllm_config.attention_config.use_non_causal:
|
||||||
return AttentionCGSupport.UNIFORM_BATCH
|
return AttentionCGSupport.UNIFORM_BATCH
|
||||||
else:
|
else:
|
||||||
return AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
|
return AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
|
||||||
|
|||||||
@@ -70,7 +70,11 @@ class DFlashProposer(SpecDecodeBaseProposer):
|
|||||||
# For DFlash we use the input embeddings to embed the mask token
|
# For DFlash we use the input embeddings to embed the mask token
|
||||||
self.parallel_drafting_hidden_state_tensor = None
|
self.parallel_drafting_hidden_state_tensor = None
|
||||||
|
|
||||||
self.dflash_causal = self.dflash_config.get("causal", False)
|
from vllm.model_executor.models.qwen3_dflash import dflash_has_any_non_causal
|
||||||
|
|
||||||
|
self.dflash_causal = not dflash_has_any_non_causal(
|
||||||
|
self.draft_model_config.hf_config
|
||||||
|
)
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def _create_draft_vllm_config(self) -> VllmConfig:
|
def _create_draft_vllm_config(self) -> VllmConfig:
|
||||||
|
|||||||
@@ -472,9 +472,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
decode_query_len=self.decode_query_len,
|
decode_query_len=self.decode_query_len,
|
||||||
lora_capture_cases=self.lora_capture_cases,
|
lora_capture_cases=self.lora_capture_cases,
|
||||||
)
|
)
|
||||||
if self.speculator is not None:
|
|
||||||
self.speculator.init_cudagraph_manager(cudagraph_mode)
|
|
||||||
|
|
||||||
check_attention_cp_compatibility(self.vllm_config)
|
check_attention_cp_compatibility(self.vllm_config)
|
||||||
if isinstance(self.speculator, DraftModelSpeculator):
|
if isinstance(self.speculator, DraftModelSpeculator):
|
||||||
# HACK(woosuk)
|
# HACK(woosuk)
|
||||||
@@ -485,6 +482,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
self.input_buffers,
|
self.input_buffers,
|
||||||
self.attn_groups,
|
self.attn_groups,
|
||||||
)
|
)
|
||||||
|
if self.speculator is not None:
|
||||||
|
# After set_attn, so the speculator can size its cudagraph mode
|
||||||
|
# to its own attention support.
|
||||||
|
self.speculator.init_cudagraph_manager(cudagraph_mode)
|
||||||
|
|
||||||
self.kv_caches: list[torch.Tensor] = []
|
self.kv_caches: list[torch.Tensor] = []
|
||||||
kv_caches_dict = init_kv_cache(
|
kv_caches_dict = init_kv_cache(
|
||||||
|
|||||||
@@ -6,11 +6,12 @@ from typing import Any
|
|||||||
import torch
|
import torch
|
||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig, replace
|
||||||
from vllm.config.compilation import CUDAGraphMode
|
from vllm.config.compilation import CUDAGraphMode
|
||||||
from vllm.forward_context import BatchDescriptor, set_forward_context
|
from vllm.forward_context import BatchDescriptor, set_forward_context
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.triton_utils import tl, triton
|
from vllm.triton_utils import tl, triton
|
||||||
|
from vllm.v1.attention.backend import AttentionCGSupport
|
||||||
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
||||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||||
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
|
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
|
||||||
@@ -19,10 +20,7 @@ from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
|
|||||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||||
from vllm.v1.worker.gpu.spec_decode.dflash.cudagraph import DFlashCudaGraphManager
|
from vllm.v1.worker.gpu.spec_decode.dflash.cudagraph import DFlashCudaGraphManager
|
||||||
from vllm.v1.worker.gpu.spec_decode.dflash.utils import (
|
from vllm.v1.worker.gpu.spec_decode.dflash.utils import load_dflash_model
|
||||||
get_dflash_causal,
|
|
||||||
load_dflash_model,
|
|
||||||
)
|
|
||||||
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
|
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
|
||||||
from vllm.v1.worker.gpu.spec_decode.utils import get_parallel_drafting_token_id
|
from vllm.v1.worker.gpu.spec_decode.utils import get_parallel_drafting_token_id
|
||||||
from vllm.v1.worker.utils import AttentionGroup
|
from vllm.v1.worker.utils import AttentionGroup
|
||||||
@@ -50,7 +48,11 @@ class DFlashSpeculator(DraftModelSpeculator):
|
|||||||
self.draft_model_config.hf_config
|
self.draft_model_config.hf_config
|
||||||
)
|
)
|
||||||
|
|
||||||
self.dflash_causal = get_dflash_causal(self.draft_model_config)
|
from vllm.model_executor.models.qwen3_dflash import dflash_has_any_non_causal
|
||||||
|
|
||||||
|
self.requires_non_causal = dflash_has_any_non_causal(
|
||||||
|
self.draft_model_config.hf_config
|
||||||
|
)
|
||||||
|
|
||||||
# Whether the anchor query position is itself a prediction. DFlash default uses
|
# Whether the anchor query position is itself a prediction. DFlash default uses
|
||||||
# the anchor as the bonus token (only mask tokens predict); DSpark samples from
|
# the anchor as the bonus token (only mask tokens predict); DSpark samples from
|
||||||
@@ -84,9 +86,32 @@ class DFlashSpeculator(DraftModelSpeculator):
|
|||||||
self.query_cudagraph_manager: DFlashCudaGraphManager | None = None
|
self.query_cudagraph_manager: DFlashCudaGraphManager | None = None
|
||||||
self.draft_kv_cache_group_id: int = -1
|
self.draft_kv_cache_group_id: int = -1
|
||||||
|
|
||||||
|
@property
|
||||||
|
def attn_vllm_config(self) -> VllmConfig:
|
||||||
|
# The draft's attention differs from the target's in causality.
|
||||||
|
return replace(
|
||||||
|
self.vllm_config,
|
||||||
|
attention_config=replace(
|
||||||
|
self.vllm_config.attention_config,
|
||||||
|
use_non_causal=self.requires_non_causal,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None:
|
def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None:
|
||||||
# PIECEWISE cudagraphs are not supported for dflash
|
wants_full = cudagraph_mode.decode_mode() == CUDAGraphMode.FULL
|
||||||
if cudagraph_mode.decode_mode() == CUDAGraphMode.FULL:
|
supports_full = (
|
||||||
|
self.attn_cg_support.min_cg_support.value
|
||||||
|
>= AttentionCGSupport.UNIFORM_BATCH.value
|
||||||
|
)
|
||||||
|
if wants_full and not supports_full:
|
||||||
|
logger.warning(
|
||||||
|
"%s draft attention (%s) does not support full CUDA graphs; "
|
||||||
|
"running the draft eagerly.",
|
||||||
|
self._speculator_name,
|
||||||
|
self.attn_cg_support.min_cg_attn_backend,
|
||||||
|
)
|
||||||
|
# PIECEWISE cudagraphs are not supported for dflash.
|
||||||
|
if wants_full and supports_full:
|
||||||
cudagraph_mode = CUDAGraphMode.FULL_DECODE_ONLY
|
cudagraph_mode = CUDAGraphMode.FULL_DECODE_ONLY
|
||||||
else:
|
else:
|
||||||
cudagraph_mode = CUDAGraphMode.NONE
|
cudagraph_mode = CUDAGraphMode.NONE
|
||||||
@@ -158,8 +183,8 @@ class DFlashSpeculator(DraftModelSpeculator):
|
|||||||
# of the kv-cache group its cache belongs to. Models that share a single group
|
# of the kv-cache group its cache belongs to. Models that share a single group
|
||||||
# leave this as None and share one context slot mapping.
|
# leave this as None and share one context slot mapping.
|
||||||
self._layer_group_idx: list[int] | None = None
|
self._layer_group_idx: list[int] | None = None
|
||||||
# Per-KV-group causal, falling back to the scalar dflash_causal.
|
# Per-KV-group causal, falling back to whether the drafter is all-causal.
|
||||||
self._group_causal: dict[int, bool] | bool = self.dflash_causal
|
self._group_causal: dict[int, bool] | bool = not self.requires_non_causal
|
||||||
if hasattr(self.model, "get_draft_kv_cache_layer_names"):
|
if hasattr(self.model, "get_draft_kv_cache_layer_names"):
|
||||||
layer_names = self.model.get_draft_kv_cache_layer_names()
|
layer_names = self.model.get_draft_kv_cache_layer_names()
|
||||||
name_to_gid = {
|
name_to_gid = {
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
|
|
||||||
from vllm.config import ModelConfig, VllmConfig, replace
|
from vllm.config import VllmConfig, replace
|
||||||
from vllm.distributed.parallel_state import get_pp_group
|
from vllm.distributed.parallel_state import get_pp_group
|
||||||
from vllm.model_executor.model_loader import get_model
|
from vllm.model_executor.model_loader import get_model
|
||||||
from vllm.v1.worker.gpu.spec_decode.eagle.utils import (
|
from vllm.v1.worker.gpu.spec_decode.eagle.utils import (
|
||||||
@@ -11,26 +11,20 @@ from vllm.v1.worker.gpu.spec_decode.eagle.utils import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def get_dflash_causal(draft_model_config: ModelConfig) -> bool:
|
|
||||||
"""Whether the DFlash draft uses causal (vs non-causal) attention."""
|
|
||||||
dflash_config = getattr(draft_model_config.hf_config, "dflash_config", None) or {}
|
|
||||||
return dflash_config.get("causal", False)
|
|
||||||
|
|
||||||
|
|
||||||
def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Module:
|
def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Module:
|
||||||
from vllm.compilation.backends import set_model_tag
|
from vllm.compilation.backends import set_model_tag
|
||||||
|
from vllm.model_executor.models.qwen3_dflash import dflash_has_any_non_causal
|
||||||
|
|
||||||
speculative_config = vllm_config.speculative_config
|
speculative_config = vllm_config.speculative_config
|
||||||
assert speculative_config is not None
|
assert speculative_config is not None
|
||||||
draft_model_config = speculative_config.draft_model_config
|
draft_model_config = speculative_config.draft_model_config
|
||||||
# Modify the attention config so that we select an attention backend that matches
|
# Select an attention backend that supports the drafter's attention: mixing
|
||||||
# the causal/non-causal mode of the dflash model.
|
# a non-causal layer onto a causal-only backend would fail.
|
||||||
causal = get_dflash_causal(draft_model_config)
|
|
||||||
draft_vllm_config = replace(
|
draft_vllm_config = replace(
|
||||||
vllm_config,
|
vllm_config,
|
||||||
attention_config=replace(
|
attention_config=replace(
|
||||||
vllm_config.attention_config,
|
vllm_config.attention_config,
|
||||||
use_non_causal=not causal,
|
use_non_causal=dflash_has_any_non_causal(draft_model_config.hf_config),
|
||||||
backend=speculative_config.attention_backend,
|
backend=speculative_config.attention_backend,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -59,8 +59,6 @@ class DSparkSpeculator(DFlashSpeculator):
|
|||||||
self.max_num_tokens, draft_hidden, dtype=self.dtype, device=device
|
self.max_num_tokens, draft_hidden, dtype=self.dtype, device=device
|
||||||
)
|
)
|
||||||
|
|
||||||
self.dflash_causal = False
|
|
||||||
|
|
||||||
self._step_cols = torch.arange(
|
self._step_cols = torch.arange(
|
||||||
self.num_speculative_steps, dtype=torch.int32, device=device
|
self.num_speculative_steps, dtype=torch.int32, device=device
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -18,14 +18,13 @@ def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
|
|||||||
draft_model_config = speculative_config.draft_model_config
|
draft_model_config = speculative_config.draft_model_config
|
||||||
|
|
||||||
from vllm.compilation.backends import set_model_tag
|
from vllm.compilation.backends import set_model_tag
|
||||||
|
from vllm.model_executor.models.qwen3_dflash import dflash_has_any_non_causal
|
||||||
|
|
||||||
# DSpark uses non-causal attention.
|
|
||||||
causal = False
|
|
||||||
draft_vllm_config = replace(
|
draft_vllm_config = replace(
|
||||||
vllm_config,
|
vllm_config,
|
||||||
attention_config=replace(
|
attention_config=replace(
|
||||||
vllm_config.attention_config,
|
vllm_config.attention_config,
|
||||||
use_non_causal=not causal,
|
use_non_causal=dflash_has_any_non_causal(draft_model_config.hf_config),
|
||||||
backend=speculative_config.attention_backend,
|
backend=speculative_config.attention_backend,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -175,6 +175,12 @@ class DraftModelSpeculator(BaseSpeculator):
|
|||||||
num_unpadded_tokens,
|
num_unpadded_tokens,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def attn_vllm_config(self) -> VllmConfig:
|
||||||
|
"""Config for the draft's attention metadata builders. Overridden by
|
||||||
|
speculators whose attention mode differs from the target's."""
|
||||||
|
return self.vllm_config
|
||||||
|
|
||||||
def set_attn(
|
def set_attn(
|
||||||
self,
|
self,
|
||||||
model_state: ModelState,
|
model_state: ModelState,
|
||||||
@@ -185,9 +191,9 @@ class DraftModelSpeculator(BaseSpeculator):
|
|||||||
) -> None:
|
) -> None:
|
||||||
self.model_state = model_state
|
self.model_state = model_state
|
||||||
self.kv_cache_config = kv_cache_config
|
self.kv_cache_config = kv_cache_config
|
||||||
self.attn_groups, _, _ = init_attn_backend(
|
self.attn_groups, self.attn_cg_support, _ = init_attn_backend(
|
||||||
kv_cache_config,
|
kv_cache_config,
|
||||||
self.vllm_config,
|
self.attn_vllm_config,
|
||||||
self.device,
|
self.device,
|
||||||
active_layer_names=self.draft_attn_layer_names,
|
active_layer_names=self.draft_attn_layer_names,
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user