423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-08-06 05:32:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix #763

Browse Source 
Former-commit-id: 370bdb6e4309db03e26cad311fb13e5cbb1fc1bf
This commit is contained in:
hiyouga 2023-09-01 23:13:05 +08:00
parent a4fd976048
commit 3b5a9c60b6
2 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/llmtuner/dsets/preprocess.py
View File

@ -34,6 +34,8 @@ def preprocess_dataset(
# build grouped texts with format `X1 X2 X3 ...` # build grouped texts with format `X1 X2 X3 ...`
if isinstance(getattr(tokenizer, "tokenizer", None), tiktoken.Encoding): if isinstance(getattr(tokenizer, "tokenizer", None), tiktoken.Encoding):
kwargs = dict(allowed_special="all") # for tiktoken tokenizer (Qwen) kwargs = dict(allowed_special="all") # for tiktoken tokenizer (Qwen)
else:
kwargs = dict(add_special_tokens=True)
if hasattr(tokenizer, "add_bos_token") and hasattr(tokenizer, "add_eos_token"): if hasattr(tokenizer, "add_bos_token") and hasattr(tokenizer, "add_eos_token"):
setattr(tokenizer, "add_bos_token", True) # for LLaMA tokenizer setattr(tokenizer, "add_bos_token", True) # for LLaMA tokenizer