423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-15 19:30:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix tokenizer #417

Browse Source
This commit is contained in:
hiyouga
2023-08-08 23:59:41 +08:00
parent caa0eda27d
commit eecc4b2131
4 changed files with 24 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/llmtuner/tuner/core/loader.py
View File

@@ -68,8 +68,6 @@ def load_model_and_tokenizer(
padding_side=model_args.padding_side,
**config_kwargs
)
if tokenizer.pad_token_id is None and tokenizer.eos_token_id is not None: # add pad token
tokenizer.pad_token = tokenizer.eos_token
if model_args.checkpoint_dir is not None and finetuning_args.finetuning_type == "full":
model_to_load = model_args.checkpoint_dir[0]