423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-15 11:20:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

improve fix tokenizer

Browse Source
This commit is contained in:
hiyouga
2024-02-09 14:53:14 +08:00
parent d0daaa01f9
commit 54ea9684ed
7 changed files with 105 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/llmtuner/data/loader.py
View File

@@ -142,7 +142,7 @@ def get_dataset(
stage: Literal["pt", "sft", "rm", "ppo"],
# split: Optional[str] = "train", # TODO: add split
) -> Union["Dataset", "IterableDataset"]:
template = get_template_and_fix_tokenizer(data_args.template, tokenizer)
template = get_template_and_fix_tokenizer(tokenizer, data_args.template)
if data_args.train_on_prompt and template.efficient_eos:
raise ValueError("Current template does not support `train_on_prompt`.")