423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-14 19:06:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix qwen eos token

Browse Source
This commit is contained in:
hiyouga
2023-08-06 13:31:17 +08:00
parent 7f18d2a335
commit e21ae01356
1 changed files with 3 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/llmtuner/tuner/core/loader.py
View File

@@ -68,7 +68,9 @@ def load_model_and_tokenizer(
padding_side=model_args.padding_side, padding_side=model_args.padding_side,
**config_kwargs **config_kwargs
) )
if tokenizer.pad_token_id is None or tokenizer.pad_token_id == 64000: # 64000 for baichuan model (older version) if tokenizer.eos_token_id is None: # fix qwen tokenizer
tokenizer.eos_token = "<|endoftext|>"
if tokenizer.pad_token_id is None: # add pad token
tokenizer.pad_token = tokenizer.eos_token tokenizer.pad_token = tokenizer.eos_token
config = AutoConfig.from_pretrained(model_args.model_name_or_path, **config_kwargs) config = AutoConfig.from_pretrained(model_args.model_name_or_path, **config_kwargs)