423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-10-15 08:08:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

alert pad_token source

Browse Source 
Former-commit-id: f26a84e0d927d2554890daf431a93652e18f4235
This commit is contained in:
hiyouga 2023-08-15 00:07:56 +08:00
parent 7f0b908de2
commit ef2ca0a827
2 changed files with 5 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/llmtuner/extras/template.py
View File

@ -204,7 +204,10 @@ def get_template_and_fix_tokenizer(
logger.info("Add eos token: {}".format(tokenizer.eos_token)) logger.info("Add eos token: {}".format(tokenizer.eos_token))
if tokenizer.pad_token_id is None: if tokenizer.pad_token_id is None:
tokenizer.pad_token = tokenizer.eos_token if tokenizer.unk_token_id is not None:
tokenizer.pad_token = tokenizer.unk_token
else:
tokenizer.pad_token = tokenizer.eos_token
logger.info("Add pad token: {}".format(tokenizer.pad_token)) logger.info("Add pad token: {}".format(tokenizer.pad_token))
tokenizer.add_special_tokens(dict(additional_special_tokens=template.stop_words)) tokenizer.add_special_tokens(dict(additional_special_tokens=template.stop_words))

2
src/llmtuner/tuner/core/loader.py
View File

@ -154,7 +154,7 @@ def load_model_and_tokenizer(
model.generate = MethodType(PreTrainedModel.generate, model) model.generate = MethodType(PreTrainedModel.generate, model)
# Fix LM head (for ChatGLM2) # Fix LM head (for ChatGLM2)
if not hasattr(model, "lm_head"): if not hasattr(model, "lm_head") and hasattr(model, "transformer"):
setattr(model, "lm_head", model.transformer.output_layer) setattr(model, "lm_head", model.transformer.output_layer)
# Register auto class to save the custom code files. # Register auto class to save the custom code files.