[Model] Support DeepSeek-V4.1-Flash (#56214)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Signed-off-by: andyluo7 <andy.luo@amd.com> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: andyluo7 <andy.luo@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Jee Jee Li
andyluo7
OpenAI Codex
zjy0516
Claude Opus 5
parent
e6821ceac9
commit
e77daef89e
@@ -42,6 +42,7 @@ from vllm.forward_context import (
|
||||
is_forward_context_available,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
|
||||
from vllm.model_executor.offloader.base import get_offloader
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import weak_ref_tensor, weak_ref_tensors
|
||||
@@ -56,6 +57,12 @@ def is_breakable_cudagraph_enabled() -> bool:
|
||||
F = TypeVar("F", bound=Callable[..., Any])
|
||||
|
||||
|
||||
def _weak_ref_capture_arg(arg: Any) -> Any:
|
||||
if isinstance(arg, QuantizedActivation):
|
||||
return arg.weak_ref()
|
||||
return weak_ref_tensor(arg)
|
||||
|
||||
|
||||
def eager_break_during_capture(fn: F) -> F:
|
||||
"""Decorator that turns a custom-op Python kernel into a "break point"
|
||||
for the breakable cudagraph capture.
|
||||
@@ -105,13 +112,8 @@ def eager_break_during_capture(fn: F) -> F:
|
||||
# Weak-ref args: strong refs in the replay lambda pin cudagraph-pool
|
||||
# slots across batch descriptors. cudagraph owns the slot, so the
|
||||
# weak_ref is safe to deref on replay.
|
||||
weak_args = tuple(
|
||||
weak_ref_tensor(a) if isinstance(a, torch.Tensor) else a for a in args
|
||||
)
|
||||
weak_kwargs = {
|
||||
k: weak_ref_tensor(v) if isinstance(v, torch.Tensor) else v
|
||||
for k, v in kwargs.items()
|
||||
}
|
||||
weak_args = tuple(_weak_ref_capture_arg(a) for a in args)
|
||||
weak_kwargs = {k: _weak_ref_capture_arg(v) for k, v in kwargs.items()}
|
||||
return capture.add_eager(lambda: fn(*weak_args, **weak_kwargs))
|
||||
|
||||
return wrapper # type: ignore[return-value]
|
||||
|
||||
Reference in New Issue
Block a user