[Bugfix] Fix OpenAI parallel sampling when using xgrammar (#11637)

Signed-off-by: mgoin <michael@neuralmagic.com>
This commit is contained in:
Michael Goin
2024-12-31 03:43:54 +00:00
committed by GitHub
parent a2a40bcd0d
commit 74fa1d123c
4 changed files with 17 additions and 13 deletions
@@ -1,6 +1,7 @@
# noqa: UP007
from __future__ import annotations
import copy
import json
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any
@@ -309,3 +310,7 @@ class XGrammarLogitsProcessor:
scores = scores.to(device_type).squeeze()
return scores
def clone(self) -> XGrammarLogitsProcessor:
"""Deepcopy due to per-sequence state in the matchers"""
return copy.deepcopy(self)