423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-08-05 21:22:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

add defaults

Browse Source 
Former-commit-id: d3844e97e387b2106a32a576a61318ecec948e23
This commit is contained in:
hiyouga 2023-08-11 13:56:26 +08:00
parent eb26bfc2ba
commit 21bf79e72b
2 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/llmtuner/dsets/preprocess.py
View File

@ -32,7 +32,7 @@ def preprocess_dataset(
def preprocess_pretrain_dataset(examples: Dict[str, List[Any]]) -> Dict[str, Any]: def preprocess_pretrain_dataset(examples: Dict[str, List[Any]]) -> Dict[str, Any]:
# build grouped texts with format `X1 X2 X3 ...` (without <eos>) # build grouped texts with format `X1 X2 X3 ...` (without <eos>)
if isinstance(getattr(tokenizer, "tokenizer"), tiktoken.Encoding): # for tiktoken tokenizer (Qwen) if isinstance(getattr(tokenizer, "tokenizer", None), tiktoken.Encoding): # for tiktoken tokenizer (Qwen)
kwargs = dict(allowed_special="all") kwargs = dict(allowed_special="all")
else: else:
kwargs = dict(add_special_tokens=False) kwargs = dict(add_special_tokens=False)

2
src/llmtuner/extras/template.py
View File

@ -123,7 +123,7 @@ class Template:
r""" r"""
Converts context to token ids. Converts context to token ids.
""" """
if isinstance(getattr(tokenizer, "tokenizer"), tiktoken.Encoding): # for tiktoken tokenizer (Qwen) if isinstance(getattr(tokenizer, "tokenizer", None), tiktoken.Encoding): # for tiktoken tokenizer (Qwen)
kwargs = dict(allowed_special="all") kwargs = dict(allowed_special="all")
else: else:
kwargs = dict(add_special_tokens=False) kwargs = dict(add_special_tokens=False)