423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-17 04:10:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

improve fix tokenizer

Browse Source
This commit is contained in:
hiyouga
2024-02-09 14:53:14 +08:00
parent d0daaa01f9
commit 54ea9684ed
7 changed files with 105 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/llmtuner/chat/chat_model.py
View File

@@ -28,7 +28,7 @@ class ChatModel:
)
self.tokenizer.padding_side = "left" if self.can_generate else "right"
self.model = dispatch_model(self.model)
self.template = get_template_and_fix_tokenizer(data_args.template, self.tokenizer)
self.template = get_template_and_fix_tokenizer(self.tokenizer, data_args.template)
def _process_args(
self,